load bash/zsh history when resh hostory is too small, fix duplicates in sesshist, fix lag at the end of hist session

cmd/daemon/main.go

@@ -31,7 +31,9 @@ func main() {
 	dir := usr.HomeDir
 	pidfilePath := filepath.Join(dir, ".resh/resh.pid")
 	configPath := filepath.Join(dir, ".config/resh.toml")
-	historyPath := filepath.Join(dir, ".resh_history.json")
+	reshHistoryPath := filepath.Join(dir, ".resh_history.json")
+	bashHistoryPath := filepath.Join(dir, ".bash_history")
+	zshHistoryPath := filepath.Join(dir, ".zsh_history")
 	logPath := filepath.Join(dir, ".resh/daemon.log")
 
 	f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
@@ -73,7 +75,7 @@ func main() {
 	if err != nil {
 		log.Fatal("Could not create pidfile", err)
 	}
-	runServer(config, historyPath)
+	runServer(config, reshHistoryPath, bashHistoryPath, zshHistoryPath)
 	log.Println("main: Removing pidfile ...")
 	err = os.Remove(pidfilePath)
 	if err != nil {

cmd/daemon/run-server.go

@@ -13,7 +13,7 @@ import (
 	"github.com/curusarn/resh/pkg/signalhandler"
 )
 
-func runServer(config cfg.Config, historyPath string) {
+func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPath string) {
 	var recordSubscribers []chan records.Record
 	var sessionInitSubscribers []chan records.Record
 	var sessionDropSubscribers []chan string
@@ -36,7 +36,12 @@ func runServer(config cfg.Config, historyPath string) {
 	sessionDropSubscribers = append(sessionDropSubscribers, histfileSessionsToDrop)
 	histfileSignals := make(chan os.Signal)
 	signalSubscribers = append(signalSubscribers, histfileSignals)
-	histfileBox := histfile.New(histfileRecords, historyPath, 10000, histfileSessionsToDrop, histfileSignals, shutdown)
+	maxHistSize := 10000 // lines
+	minHistSizeKB := 100 // roughly lines
+	histfileBox := histfile.New(histfileRecords, histfileSessionsToDrop,
+		reshHistoryPath, bashHistoryPath, zshHistoryPath,
+		maxHistSize, minHistSizeKB,
+		histfileSignals, shutdown)
 
 	// sesshist New
 	sesshistDispatch := sesshist.NewDispatch(sesshistSessionsToInit, sesshistSessionsToDrop,

pkg/histfile/histfile.go

@@ -3,6 +3,7 @@ package histfile
 import (
 	"encoding/json"
 	"log"
+	"math"
 	"os"
 	"strconv"
 	"sync"
@@ -20,30 +21,68 @@ type Histfile struct {
 	recentMutex   sync.Mutex
 	recentRecords []records.Record
 
-	cmdLines histlist.Histlist
+	// NOTE: we have separate histories which only differ if there was not enough resh_history
+	//			resh_history itself is common for both bash and zsh
+	bashCmdLines histlist.Histlist
+	zshCmdLines  histlist.Histlist
 }
 
-// New creates new histfile and runs two gorutines on it
+// New creates new histfile and runs its gorutines
-func New(input chan records.Record, historyPath string, initHistSize int, sessionsToDrop chan string,
+func New(input chan records.Record, sessionsToDrop chan string,
+	reshHistoryPath string, bashHistoryPath string, zshHistoryPath string,
+	maxInitHistSize int, minInitHistSizeKB int,
 	signals chan os.Signal, shutdownDone chan string) *Histfile {
 
 	hf := Histfile{
-		sessions:    map[string]records.Record{},
+		sessions:     map[string]records.Record{},
-		historyPath: historyPath,
+		historyPath:  reshHistoryPath,
-		cmdLines:    histlist.New(),
+		bashCmdLines: histlist.New(),
+		zshCmdLines:  histlist.New(),
 	}
-	go hf.loadHistory(initHistSize)
+	go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB)
 	go hf.writer(input, signals, shutdownDone)
 	go hf.sessionGC(sessionsToDrop)
 	return &hf
 }
 
-func (h *Histfile) loadHistory(initHistSize int) {
+// loadsHistory from resh_history and if there is not enough of it also load native shell histories
+func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHistSize, minInitHistSizeKB int) {
 	h.recentMutex.Lock()
 	defer h.recentMutex.Unlock()
-	log.Println("histfile: Loading history from file ...")
+	log.Println("histfile: Checking if resh_history is large enough ...")
-	h.cmdLines = records.LoadCmdLinesFromFile(h.historyPath, initHistSize)
+	fi, err := os.Stat(h.historyPath)
-	log.Println("histfile: History loaded - cmdLine count:", len(h.cmdLines.List))
+	var size int
+	if err != nil {
+		log.Println("histfile ERROR: failed to stat resh_history file:", err)
+	} else {
+		size = int(fi.Size())
+	}
+	useNativeHistories := false
+	if size/1024 < minInitHistSizeKB {
+		useNativeHistories = true
+		log.Println("histfile WARN: resh_history is too small - loading native bash and zsh history ...")
+		h.bashCmdLines = records.LoadCmdLinesFromBashFile(bashHistoryPath)
+		log.Println("histfile: bash history loaded - cmdLine count:", len(h.bashCmdLines.List))
+		h.zshCmdLines = records.LoadCmdLinesFromZshFile(zshHistoryPath)
+		log.Println("histfile: zsh history loaded - cmdLine count:", len(h.zshCmdLines.List))
+		// no maxInitHistSize when using native histories
+		maxInitHistSize = math.MaxInt32
+	}
+	log.Println("histfile: Loading resh history from file ...")
+	reshCmdLines := histlist.New()
+	// NOTE: keeping this weird interface for now because we might use it in the future
+	//			when we only load bash or zsh history
+	records.LoadCmdLinesFromFile(&reshCmdLines, h.historyPath, maxInitHistSize)
+	log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List))
+	if useNativeHistories == false {
+		h.bashCmdLines = reshCmdLines
+		h.zshCmdLines = histlist.Copy(reshCmdLines)
+		return
+	}
+	h.bashCmdLines.AddHistlist(reshCmdLines)
+	log.Println("histfile: bash history + resh history - cmdLine count:", len(h.bashCmdLines.List))
+	h.zshCmdLines.AddHistlist(reshCmdLines)
+	log.Println("histfile: zsh history + resh history - cmdLine count:", len(h.zshCmdLines.List))
 }
 
 // sessionGC reads sessionIDs from channel and deletes them from histfile struct
@@ -124,12 +163,8 @@ func (h *Histfile) mergeAndWriteRecord(part1, part2 records.Record) {
 		defer h.recentMutex.Unlock()
 		h.recentRecords = append(h.recentRecords, part1)
 		cmdLine := part1.CmdLine
-		idx, found := h.cmdLines.LastIndex[cmdLine]
+		h.bashCmdLines.AddCmdLine(cmdLine)
-		if found {
+		h.zshCmdLines.AddCmdLine(cmdLine)
-			h.cmdLines.List = append(h.cmdLines.List[:idx], h.cmdLines.List[idx+1:]...)
-		}
-		h.cmdLines.LastIndex[cmdLine] = len(h.cmdLines.List)
-		h.cmdLines.List = append(h.cmdLines.List, cmdLine)
 	}()
 
 	writeRecord(part1, h.historyPath)
@@ -156,11 +191,21 @@ func writeRecord(rec records.Record, outputPath string) {
 }
 
 // GetRecentCmdLines returns recent cmdLines
-func (h *Histfile) GetRecentCmdLines(limit int) histlist.Histlist {
+func (h *Histfile) GetRecentCmdLines(shell string, limit int) histlist.Histlist {
+	// NOTE: limit does nothing atm
 	h.recentMutex.Lock()
 	defer h.recentMutex.Unlock()
 	log.Println("histfile: History requested ...")
-	hl := histlist.Copy(h.cmdLines)
+	var hl histlist.Histlist
-	log.Println("histfile: History copied - cmdLine count:", len(hl.List))
+	if shell == "bash" {
+		hl = histlist.Copy(h.bashCmdLines)
+		log.Println("histfile: history copied (bash) - cmdLine count:", len(hl.List))
+		return hl
+	}
+	if shell != "zsh" {
+		log.Println("histfile ERROR: Unknown shell: ", shell)
+	}
+	hl = histlist.Copy(h.zshCmdLines)
+	log.Println("histfile: history copied (zsh) - cmdLine count:", len(hl.List))
 	return hl
 }

pkg/histlist/histlist.go

@@ -1,5 +1,7 @@
 package histlist
 
+import "log"
+
 // Histlist is a deduplicated list of cmdLines
 type Histlist struct {
 	// list of commands lines (deduplicated)
@@ -25,3 +27,38 @@ func Copy(hl Histlist) Histlist {
 	}
 	return newHl
 }
+
+// AddCmdLine to the histlist
+func (h *Histlist) AddCmdLine(cmdLine string) {
+	lenBefore := len(h.List)
+	// lookup
+	idx, found := h.LastIndex[cmdLine]
+	if found {
+		// remove duplicate
+		if cmdLine != h.List[idx] {
+			log.Println("histlist ERROR: Adding cmdLine:", cmdLine, " != LastIndex[cmdLine]:", h.List[idx])
+		}
+		h.List = append(h.List[:idx], h.List[idx+1:]...)
+		// idx++
+		for idx < len(h.List) {
+			cmdLn := h.List[idx]
+			h.LastIndex[cmdLn]--
+			if idx != h.LastIndex[cmdLn] {
+				log.Println("histlist ERROR: Shifting LastIndex idx:", idx, " != LastIndex[cmdLn]:", h.LastIndex[cmdLn])
+			}
+			idx++
+		}
+	}
+	// update last index
+	h.LastIndex[cmdLine] = len(h.List)
+	// append new cmdline
+	h.List = append(h.List, cmdLine)
+	log.Println("histlist: Added cmdLine:", cmdLine, "; history length:", lenBefore, "->", len(h.List))
+}
+
+// AddHistlist contents of another histlist to this histlist
+func (h *Histlist) AddHistlist(h2 Histlist) {
+	for _, cmdLine := range h2.List {
+		h.AddCmdLine(cmdLine)
+	}
+}

pkg/records/records.go

@@ -455,9 +455,10 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 {
 	return dist
 }
 
-// LoadCmdLinesFromFile loads limit cmdlines from file
+// LoadCmdLinesFromFile loads cmdlines from file
-func LoadCmdLinesFromFile(fname string, limit int) histlist.Histlist {
+func LoadCmdLinesFromFile(hl *histlist.Histlist, fname string, limit int) {
 	recs := LoadFromFile(fname, limit*3) // assume that at least 1/3 of commands is unique
+	// go from bottom and deduplicate
 	var cmdLines []string
 	cmdLinesSet := map[string]bool{}
 	for i := len(recs) - 1; i >= 0; i-- {
@@ -471,24 +472,24 @@ func LoadCmdLinesFromFile(fname string, limit int) histlist.Histlist {
 			break
 		}
 	}
-	hl := histlist.New()
+	// add everything to histlist
-	hl.List = cmdLines
+	for _, cmdLine := range cmdLines {
-	for idx, cmdLine := range cmdLines {
+		hl.AddCmdLine(cmdLine)
-		hl.LastIndex[cmdLine] = idx
 	}
-	return hl
 }
 
-// LoadFromFile loads at most 'limit' records from 'fname' file
+// LoadFromFile loads records from 'fname' file
 func LoadFromFile(fname string, limit int) []Record {
 	// NOTE: limit does nothing atm
+	var recs []Record
 	file, err := os.Open(fname)
 	if err != nil {
-		log.Fatal("Open() resh history file error:", err)
+		log.Println("Open() resh history file error:", err)
+		log.Println("WARN: Skipping reading resh history!")
+		return recs
 	}
 	defer file.Close()
 
-	var recs []Record
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		record := Record{}
@@ -507,3 +508,70 @@ func LoadFromFile(fname string, limit int) []Record {
 	}
 	return recs
 }
+
+// LoadCmdLinesFromZshFile loads cmdlines from zsh history file
+func LoadCmdLinesFromZshFile(fname string) histlist.Histlist {
+	file, err := os.Open(fname)
+	if err != nil {
+		log.Fatal("Open() resh history file error:", err)
+	}
+	defer file.Close()
+
+	hl := histlist.New()
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// trim newline
+		line = strings.TrimRight(line, "\n")
+		var cmd string
+		// zsh format EXTENDED_HISTORY
+		// : 1576270617:0;make install
+		// zsh format no EXTENDED_HISTORY
+		// make install
+		if len(line) == 0 {
+			// skip empty
+			continue
+		}
+		if strings.Contains(line, ":") && strings.Contains(line, ";") &&
+			len(strings.Split(line, ":")) >= 3 && len(strings.Split(line, ";")) >= 2 {
+			// contains at least 2x ':' and 1x ';' => assume EXTENDED_HISTORY
+			cmd = strings.Split(line, ";")[1]
+		} else {
+			cmd = line
+		}
+		hl.AddCmdLine(cmd)
+	}
+	return hl
+}
+
+// LoadCmdLinesFromBashFile loads cmdlines from bash history file
+func LoadCmdLinesFromBashFile(fname string) histlist.Histlist {
+	file, err := os.Open(fname)
+	if err != nil {
+		log.Fatal("Open() resh history file error:", err)
+	}
+	defer file.Close()
+
+	hl := histlist.New()
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		// trim newline
+		line = strings.TrimRight(line, "\n")
+		// trim spaces from left
+		line = strings.TrimLeft(line, " ")
+		// bash format (two lines)
+		// #1576199174
+		// make install
+		if strings.HasPrefix(line, "#") {
+			// is either timestamp or comment => skip
+			continue
+		}
+		if len(line) == 0 {
+			// skip empty
+			continue
+		}
+		hl.AddCmdLine(line)
+	}
+	return hl
+}

pkg/sesshist/sesshist.go

@@ -40,7 +40,7 @@ func (s *Dispatch) sessionInitializer(sessionsToInit chan records.Record) {
 	for {
 		record := <-sessionsToInit
 		log.Println("sesshist: got session to init - " + record.SessionID)
-		s.initSession(record.SessionID)
+		s.initSession(record.SessionID, record.Shell)
 	}
 }
 
@@ -64,7 +64,7 @@ func (s *Dispatch) recordAdder(recordsToAdd chan records.Record) {
 }
 
 // InitSession struct
-func (s *Dispatch) initSession(sessionID string) error {
+func (s *Dispatch) initSession(sessionID string, shell string) error {
 	log.Println("sesshist: initializing session - " + sessionID)
 	s.mutex.RLock()
 	_, found := s.sessions[sessionID]
@@ -75,7 +75,7 @@ func (s *Dispatch) initSession(sessionID string) error {
 	}
 
 	log.Println("sesshist: loading history to populate session - " + sessionID)
-	historyCmdLines := s.history.GetRecentCmdLines(s.historyInitSize)
+	historyCmdLines := s.history.GetRecentCmdLines(shell, s.historyInitSize)
 
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
@@ -113,29 +113,15 @@ func (s *Dispatch) addRecentRecord(sessionID string, record records.Record) erro
 	s.mutex.RUnlock()
 
 	if found == false {
-		log.Println("sesshist ERROR: addRecontRecord(): No session history for SessionID " + sessionID + " - creating session history.")
+		log.Println("sesshist ERROR: addRecentRecord(): No session history for SessionID " + sessionID + " - creating session history.")
-		s.initSession(sessionID)
+		s.initSession(sessionID, record.Shell)
 		return s.addRecentRecord(sessionID, record)
 	}
 	log.Println("sesshist: RLocking session lock (w/ defer) ...")
 	session.mutex.Lock()
 	defer session.mutex.Unlock()
 	session.recentRecords = append(session.recentRecords, record)
-	// remove previous occurance of record
+	session.recentCmdLines.AddCmdLine(record.CmdLine)
-	log.Println("sesshist: Looking for duplicate cmdLine ...")
-	cmdLine := record.CmdLine
-	// trim spaces to have less duplicates in the sesshist
-	cmdLine = strings.TrimRight(cmdLine, " ")
-	idx, found := session.recentCmdLines.LastIndex[cmdLine]
-	if found {
-		log.Println("sesshist: Removing duplicate cmdLine at index:", idx, " out of", len(session.recentCmdLines.List), "...")
-		session.recentCmdLines.List = append(session.recentCmdLines.List[:idx], session.recentCmdLines.List[idx+1:]...)
-	}
-	log.Println("sesshist: Updating last index ...")
-	session.recentCmdLines.LastIndex[cmdLine] = len(session.recentCmdLines.List)
-	// append new record
-	log.Println("sesshist: Appending cmdLine ...")
-	session.recentCmdLines.List = append(session.recentCmdLines.List, cmdLine)
 	log.Println("sesshist: record:", record.CmdLine, "; added to session:", sessionID,
 		"; session len:", len(session.recentCmdLines.List), "; session len (records):", len(session.recentRecords))
 	return nil

scripts/hooks.sh

@@ -1,6 +1,7 @@
 
 __resh_reset_variables() {
     __RESH_HISTNO=0
+    __RESH_HISTNO_MAX=""
     __RESH_HISTNO_ZERO_LINE=""
     __RESH_HIST_PREV_LINE=""
     __RESH_HIST_RECALL_ACTIONS=""

scripts/widgets.sh

@@ -37,8 +37,12 @@ __resh_widget_arrow_up() {
     __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;arrow_up:$__RESH_PREFIX"
     # increment histno
     __RESH_HISTNO=$((__RESH_HISTNO+1))
-    # back at histno == 0 => restore original line
+    if [ "${#__RESH_HISTNO_MAX}" -gt 0 ] && [ "${__RESH_HISTNO}" -gt "${__RESH_HISTNO_MAX}" ]; then
-    if [ "$__RESH_HISTNO" -eq 0 ]; then
+        # end of the session -> don't recall, do nothing
+        # fix histno
+        __RESH_HISTNO=$((__RESH_HISTNO-1))
+    elif [ "$__RESH_HISTNO" -eq 0 ]; then
+        # back at histno == 0 => restore original line
         BUFFER=$__RESH_HISTNO_ZERO_LINE
     else
         # run recall
@@ -46,7 +50,12 @@ __resh_widget_arrow_up() {
         NEW_BUFFER="$(__resh_collect --recall --prefix-search "$__RESH_PREFIX" 2> ~/.resh/arrow_up_last_run_out.txt)"
         # IF new buffer in non-empty THEN use the new buffer ELSE revert histno change
         # shellcheck disable=SC2015
-        [ "${#NEW_BUFFER}" -gt 0 ] && BUFFER=$NEW_BUFFER || __RESH_HISTNO=$((__RESH_HISTNO-1))
+        if [ "${#NEW_BUFFER}" -gt 0 ]; then
+            BUFFER=$NEW_BUFFER
+        else
+            __RESH_HISTNO=$((__RESH_HISTNO-1))
+            __RESH_HISTNO_MAX=$__RESH_HISTNO
+        fi
     fi
     # run post helper
     __resh_helper_arrow_post