From aba197561a2106bf706febed856e639cb8582444 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 14 Dec 2019 15:36:53 +0100 Subject: [PATCH] load bash/zsh history when resh hostory is too small, fix duplicates in sesshist, fix lag at the end of hist session --- cmd/daemon/main.go | 6 ++- cmd/daemon/run-server.go | 9 +++- pkg/histfile/histfile.go | 85 +++++++++++++++++++++++++++++--------- pkg/histlist/histlist.go | 37 +++++++++++++++++ pkg/records/records.go | 88 +++++++++++++++++++++++++++++++++++----- pkg/sesshist/sesshist.go | 26 +++--------- scripts/hooks.sh | 1 + scripts/widgets.sh | 15 +++++-- 8 files changed, 210 insertions(+), 57 deletions(-) diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index 5591399..8045c5c 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -31,7 +31,9 @@ func main() { dir := usr.HomeDir pidfilePath := filepath.Join(dir, ".resh/resh.pid") configPath := filepath.Join(dir, ".config/resh.toml") - historyPath := filepath.Join(dir, ".resh_history.json") + reshHistoryPath := filepath.Join(dir, ".resh_history.json") + bashHistoryPath := filepath.Join(dir, ".bash_history") + zshHistoryPath := filepath.Join(dir, ".zsh_history") logPath := filepath.Join(dir, ".resh/daemon.log") f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) @@ -73,7 +75,7 @@ func main() { if err != nil { log.Fatal("Could not create pidfile", err) } - runServer(config, historyPath) + runServer(config, reshHistoryPath, bashHistoryPath, zshHistoryPath) log.Println("main: Removing pidfile ...") err = os.Remove(pidfilePath) if err != nil { diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index df25adf..0be0ec8 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -13,7 +13,7 @@ import ( "github.com/curusarn/resh/pkg/signalhandler" ) -func runServer(config cfg.Config, historyPath string) { +func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPath string) { var recordSubscribers []chan records.Record var sessionInitSubscribers []chan records.Record var sessionDropSubscribers []chan string @@ -36,7 +36,12 @@ func runServer(config cfg.Config, historyPath string) { sessionDropSubscribers = append(sessionDropSubscribers, histfileSessionsToDrop) histfileSignals := make(chan os.Signal) signalSubscribers = append(signalSubscribers, histfileSignals) - histfileBox := histfile.New(histfileRecords, historyPath, 10000, histfileSessionsToDrop, histfileSignals, shutdown) + maxHistSize := 10000 // lines + minHistSizeKB := 100 // roughly lines + histfileBox := histfile.New(histfileRecords, histfileSessionsToDrop, + reshHistoryPath, bashHistoryPath, zshHistoryPath, + maxHistSize, minHistSizeKB, + histfileSignals, shutdown) // sesshist New sesshistDispatch := sesshist.NewDispatch(sesshistSessionsToInit, sesshistSessionsToDrop, diff --git a/pkg/histfile/histfile.go b/pkg/histfile/histfile.go index 74f125d..d2740d6 100644 --- a/pkg/histfile/histfile.go +++ b/pkg/histfile/histfile.go @@ -3,6 +3,7 @@ package histfile import ( "encoding/json" "log" + "math" "os" "strconv" "sync" @@ -20,30 +21,68 @@ type Histfile struct { recentMutex sync.Mutex recentRecords []records.Record - cmdLines histlist.Histlist + // NOTE: we have separate histories which only differ if there was not enough resh_history + // resh_history itself is common for both bash and zsh + bashCmdLines histlist.Histlist + zshCmdLines histlist.Histlist } -// New creates new histfile and runs two gorutines on it -func New(input chan records.Record, historyPath string, initHistSize int, sessionsToDrop chan string, +// New creates new histfile and runs its gorutines +func New(input chan records.Record, sessionsToDrop chan string, + reshHistoryPath string, bashHistoryPath string, zshHistoryPath string, + maxInitHistSize int, minInitHistSizeKB int, signals chan os.Signal, shutdownDone chan string) *Histfile { hf := Histfile{ - sessions: map[string]records.Record{}, - historyPath: historyPath, - cmdLines: histlist.New(), + sessions: map[string]records.Record{}, + historyPath: reshHistoryPath, + bashCmdLines: histlist.New(), + zshCmdLines: histlist.New(), } - go hf.loadHistory(initHistSize) + go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB) go hf.writer(input, signals, shutdownDone) go hf.sessionGC(sessionsToDrop) return &hf } -func (h *Histfile) loadHistory(initHistSize int) { +// loadsHistory from resh_history and if there is not enough of it also load native shell histories +func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHistSize, minInitHistSizeKB int) { h.recentMutex.Lock() defer h.recentMutex.Unlock() - log.Println("histfile: Loading history from file ...") - h.cmdLines = records.LoadCmdLinesFromFile(h.historyPath, initHistSize) - log.Println("histfile: History loaded - cmdLine count:", len(h.cmdLines.List)) + log.Println("histfile: Checking if resh_history is large enough ...") + fi, err := os.Stat(h.historyPath) + var size int + if err != nil { + log.Println("histfile ERROR: failed to stat resh_history file:", err) + } else { + size = int(fi.Size()) + } + useNativeHistories := false + if size/1024 < minInitHistSizeKB { + useNativeHistories = true + log.Println("histfile WARN: resh_history is too small - loading native bash and zsh history ...") + h.bashCmdLines = records.LoadCmdLinesFromBashFile(bashHistoryPath) + log.Println("histfile: bash history loaded - cmdLine count:", len(h.bashCmdLines.List)) + h.zshCmdLines = records.LoadCmdLinesFromZshFile(zshHistoryPath) + log.Println("histfile: zsh history loaded - cmdLine count:", len(h.zshCmdLines.List)) + // no maxInitHistSize when using native histories + maxInitHistSize = math.MaxInt32 + } + log.Println("histfile: Loading resh history from file ...") + reshCmdLines := histlist.New() + // NOTE: keeping this weird interface for now because we might use it in the future + // when we only load bash or zsh history + records.LoadCmdLinesFromFile(&reshCmdLines, h.historyPath, maxInitHistSize) + log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List)) + if useNativeHistories == false { + h.bashCmdLines = reshCmdLines + h.zshCmdLines = histlist.Copy(reshCmdLines) + return + } + h.bashCmdLines.AddHistlist(reshCmdLines) + log.Println("histfile: bash history + resh history - cmdLine count:", len(h.bashCmdLines.List)) + h.zshCmdLines.AddHistlist(reshCmdLines) + log.Println("histfile: zsh history + resh history - cmdLine count:", len(h.zshCmdLines.List)) } // sessionGC reads sessionIDs from channel and deletes them from histfile struct @@ -124,12 +163,8 @@ func (h *Histfile) mergeAndWriteRecord(part1, part2 records.Record) { defer h.recentMutex.Unlock() h.recentRecords = append(h.recentRecords, part1) cmdLine := part1.CmdLine - idx, found := h.cmdLines.LastIndex[cmdLine] - if found { - h.cmdLines.List = append(h.cmdLines.List[:idx], h.cmdLines.List[idx+1:]...) - } - h.cmdLines.LastIndex[cmdLine] = len(h.cmdLines.List) - h.cmdLines.List = append(h.cmdLines.List, cmdLine) + h.bashCmdLines.AddCmdLine(cmdLine) + h.zshCmdLines.AddCmdLine(cmdLine) }() writeRecord(part1, h.historyPath) @@ -156,11 +191,21 @@ func writeRecord(rec records.Record, outputPath string) { } // GetRecentCmdLines returns recent cmdLines -func (h *Histfile) GetRecentCmdLines(limit int) histlist.Histlist { +func (h *Histfile) GetRecentCmdLines(shell string, limit int) histlist.Histlist { + // NOTE: limit does nothing atm h.recentMutex.Lock() defer h.recentMutex.Unlock() log.Println("histfile: History requested ...") - hl := histlist.Copy(h.cmdLines) - log.Println("histfile: History copied - cmdLine count:", len(hl.List)) + var hl histlist.Histlist + if shell == "bash" { + hl = histlist.Copy(h.bashCmdLines) + log.Println("histfile: history copied (bash) - cmdLine count:", len(hl.List)) + return hl + } + if shell != "zsh" { + log.Println("histfile ERROR: Unknown shell: ", shell) + } + hl = histlist.Copy(h.zshCmdLines) + log.Println("histfile: history copied (zsh) - cmdLine count:", len(hl.List)) return hl } diff --git a/pkg/histlist/histlist.go b/pkg/histlist/histlist.go index fc6f9b0..0f5f621 100644 --- a/pkg/histlist/histlist.go +++ b/pkg/histlist/histlist.go @@ -1,5 +1,7 @@ package histlist +import "log" + // Histlist is a deduplicated list of cmdLines type Histlist struct { // list of commands lines (deduplicated) @@ -25,3 +27,38 @@ func Copy(hl Histlist) Histlist { } return newHl } + +// AddCmdLine to the histlist +func (h *Histlist) AddCmdLine(cmdLine string) { + lenBefore := len(h.List) + // lookup + idx, found := h.LastIndex[cmdLine] + if found { + // remove duplicate + if cmdLine != h.List[idx] { + log.Println("histlist ERROR: Adding cmdLine:", cmdLine, " != LastIndex[cmdLine]:", h.List[idx]) + } + h.List = append(h.List[:idx], h.List[idx+1:]...) + // idx++ + for idx < len(h.List) { + cmdLn := h.List[idx] + h.LastIndex[cmdLn]-- + if idx != h.LastIndex[cmdLn] { + log.Println("histlist ERROR: Shifting LastIndex idx:", idx, " != LastIndex[cmdLn]:", h.LastIndex[cmdLn]) + } + idx++ + } + } + // update last index + h.LastIndex[cmdLine] = len(h.List) + // append new cmdline + h.List = append(h.List, cmdLine) + log.Println("histlist: Added cmdLine:", cmdLine, "; history length:", lenBefore, "->", len(h.List)) +} + +// AddHistlist contents of another histlist to this histlist +func (h *Histlist) AddHistlist(h2 Histlist) { + for _, cmdLine := range h2.List { + h.AddCmdLine(cmdLine) + } +} diff --git a/pkg/records/records.go b/pkg/records/records.go index 4ff8c19..29477fe 100644 --- a/pkg/records/records.go +++ b/pkg/records/records.go @@ -455,9 +455,10 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { return dist } -// LoadCmdLinesFromFile loads limit cmdlines from file -func LoadCmdLinesFromFile(fname string, limit int) histlist.Histlist { +// LoadCmdLinesFromFile loads cmdlines from file +func LoadCmdLinesFromFile(hl *histlist.Histlist, fname string, limit int) { recs := LoadFromFile(fname, limit*3) // assume that at least 1/3 of commands is unique + // go from bottom and deduplicate var cmdLines []string cmdLinesSet := map[string]bool{} for i := len(recs) - 1; i >= 0; i-- { @@ -471,24 +472,24 @@ func LoadCmdLinesFromFile(fname string, limit int) histlist.Histlist { break } } - hl := histlist.New() - hl.List = cmdLines - for idx, cmdLine := range cmdLines { - hl.LastIndex[cmdLine] = idx + // add everything to histlist + for _, cmdLine := range cmdLines { + hl.AddCmdLine(cmdLine) } - return hl } -// LoadFromFile loads at most 'limit' records from 'fname' file +// LoadFromFile loads records from 'fname' file func LoadFromFile(fname string, limit int) []Record { // NOTE: limit does nothing atm + var recs []Record file, err := os.Open(fname) if err != nil { - log.Fatal("Open() resh history file error:", err) + log.Println("Open() resh history file error:", err) + log.Println("WARN: Skipping reading resh history!") + return recs } defer file.Close() - var recs []Record scanner := bufio.NewScanner(file) for scanner.Scan() { record := Record{} @@ -507,3 +508,70 @@ func LoadFromFile(fname string, limit int) []Record { } return recs } + +// LoadCmdLinesFromZshFile loads cmdlines from zsh history file +func LoadCmdLinesFromZshFile(fname string) histlist.Histlist { + file, err := os.Open(fname) + if err != nil { + log.Fatal("Open() resh history file error:", err) + } + defer file.Close() + + hl := histlist.New() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + // trim newline + line = strings.TrimRight(line, "\n") + var cmd string + // zsh format EXTENDED_HISTORY + // : 1576270617:0;make install + // zsh format no EXTENDED_HISTORY + // make install + if len(line) == 0 { + // skip empty + continue + } + if strings.Contains(line, ":") && strings.Contains(line, ";") && + len(strings.Split(line, ":")) >= 3 && len(strings.Split(line, ";")) >= 2 { + // contains at least 2x ':' and 1x ';' => assume EXTENDED_HISTORY + cmd = strings.Split(line, ";")[1] + } else { + cmd = line + } + hl.AddCmdLine(cmd) + } + return hl +} + +// LoadCmdLinesFromBashFile loads cmdlines from bash history file +func LoadCmdLinesFromBashFile(fname string) histlist.Histlist { + file, err := os.Open(fname) + if err != nil { + log.Fatal("Open() resh history file error:", err) + } + defer file.Close() + + hl := histlist.New() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + // trim newline + line = strings.TrimRight(line, "\n") + // trim spaces from left + line = strings.TrimLeft(line, " ") + // bash format (two lines) + // #1576199174 + // make install + if strings.HasPrefix(line, "#") { + // is either timestamp or comment => skip + continue + } + if len(line) == 0 { + // skip empty + continue + } + hl.AddCmdLine(line) + } + return hl +} diff --git a/pkg/sesshist/sesshist.go b/pkg/sesshist/sesshist.go index 1d6b32b..0aefa91 100644 --- a/pkg/sesshist/sesshist.go +++ b/pkg/sesshist/sesshist.go @@ -40,7 +40,7 @@ func (s *Dispatch) sessionInitializer(sessionsToInit chan records.Record) { for { record := <-sessionsToInit log.Println("sesshist: got session to init - " + record.SessionID) - s.initSession(record.SessionID) + s.initSession(record.SessionID, record.Shell) } } @@ -64,7 +64,7 @@ func (s *Dispatch) recordAdder(recordsToAdd chan records.Record) { } // InitSession struct -func (s *Dispatch) initSession(sessionID string) error { +func (s *Dispatch) initSession(sessionID string, shell string) error { log.Println("sesshist: initializing session - " + sessionID) s.mutex.RLock() _, found := s.sessions[sessionID] @@ -75,7 +75,7 @@ func (s *Dispatch) initSession(sessionID string) error { } log.Println("sesshist: loading history to populate session - " + sessionID) - historyCmdLines := s.history.GetRecentCmdLines(s.historyInitSize) + historyCmdLines := s.history.GetRecentCmdLines(shell, s.historyInitSize) s.mutex.Lock() defer s.mutex.Unlock() @@ -113,29 +113,15 @@ func (s *Dispatch) addRecentRecord(sessionID string, record records.Record) erro s.mutex.RUnlock() if found == false { - log.Println("sesshist ERROR: addRecontRecord(): No session history for SessionID " + sessionID + " - creating session history.") - s.initSession(sessionID) + log.Println("sesshist ERROR: addRecentRecord(): No session history for SessionID " + sessionID + " - creating session history.") + s.initSession(sessionID, record.Shell) return s.addRecentRecord(sessionID, record) } log.Println("sesshist: RLocking session lock (w/ defer) ...") session.mutex.Lock() defer session.mutex.Unlock() session.recentRecords = append(session.recentRecords, record) - // remove previous occurance of record - log.Println("sesshist: Looking for duplicate cmdLine ...") - cmdLine := record.CmdLine - // trim spaces to have less duplicates in the sesshist - cmdLine = strings.TrimRight(cmdLine, " ") - idx, found := session.recentCmdLines.LastIndex[cmdLine] - if found { - log.Println("sesshist: Removing duplicate cmdLine at index:", idx, " out of", len(session.recentCmdLines.List), "...") - session.recentCmdLines.List = append(session.recentCmdLines.List[:idx], session.recentCmdLines.List[idx+1:]...) - } - log.Println("sesshist: Updating last index ...") - session.recentCmdLines.LastIndex[cmdLine] = len(session.recentCmdLines.List) - // append new record - log.Println("sesshist: Appending cmdLine ...") - session.recentCmdLines.List = append(session.recentCmdLines.List, cmdLine) + session.recentCmdLines.AddCmdLine(record.CmdLine) log.Println("sesshist: record:", record.CmdLine, "; added to session:", sessionID, "; session len:", len(session.recentCmdLines.List), "; session len (records):", len(session.recentRecords)) return nil diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 4963f95..8da69c3 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -1,6 +1,7 @@ __resh_reset_variables() { __RESH_HISTNO=0 + __RESH_HISTNO_MAX="" __RESH_HISTNO_ZERO_LINE="" __RESH_HIST_PREV_LINE="" __RESH_HIST_RECALL_ACTIONS="" diff --git a/scripts/widgets.sh b/scripts/widgets.sh index 82761bd..24143b8 100644 --- a/scripts/widgets.sh +++ b/scripts/widgets.sh @@ -37,8 +37,12 @@ __resh_widget_arrow_up() { __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;arrow_up:$__RESH_PREFIX" # increment histno __RESH_HISTNO=$((__RESH_HISTNO+1)) - # back at histno == 0 => restore original line - if [ "$__RESH_HISTNO" -eq 0 ]; then + if [ "${#__RESH_HISTNO_MAX}" -gt 0 ] && [ "${__RESH_HISTNO}" -gt "${__RESH_HISTNO_MAX}" ]; then + # end of the session -> don't recall, do nothing + # fix histno + __RESH_HISTNO=$((__RESH_HISTNO-1)) + elif [ "$__RESH_HISTNO" -eq 0 ]; then + # back at histno == 0 => restore original line BUFFER=$__RESH_HISTNO_ZERO_LINE else # run recall @@ -46,7 +50,12 @@ __resh_widget_arrow_up() { NEW_BUFFER="$(__resh_collect --recall --prefix-search "$__RESH_PREFIX" 2> ~/.resh/arrow_up_last_run_out.txt)" # IF new buffer in non-empty THEN use the new buffer ELSE revert histno change # shellcheck disable=SC2015 - [ "${#NEW_BUFFER}" -gt 0 ] && BUFFER=$NEW_BUFFER || __RESH_HISTNO=$((__RESH_HISTNO-1)) + if [ "${#NEW_BUFFER}" -gt 0 ]; then + BUFFER=$NEW_BUFFER + else + __RESH_HISTNO=$((__RESH_HISTNO-1)) + __RESH_HISTNO_MAX=$__RESH_HISTNO + fi fi # run post helper __resh_helper_arrow_post