diff --git a/VERSION b/VERSION index 0664a8f..6085e94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.6 +1.2.1 diff --git a/cmd/collect/main.go b/cmd/collect/main.go index 55235a8..5699ac6 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -56,6 +56,7 @@ func main() { // recall metadata recallActions := flag.String("recall-actions", "", "recall actions that took place before executing the command") + recallStrategy := flag.String("recall-strategy", "", "recall strategy used during recall actions") // posix variables cols := flag.String("cols", "-1", "$COLUMNS") @@ -124,10 +125,6 @@ func main() { ")") os.Exit(3) } - if *recallHistno != 0 && *recall == false { - log.Println("Option '--recall' only works with '--histno' option - exiting!") - os.Exit(4) - } if *recallPrefix != "" && *recall == false { log.Println("Option '--prefix-search' only works with '--recall' option - exiting!") os.Exit(4) @@ -232,8 +229,9 @@ func main() { ReshVersion: Version, ReshRevision: Revision, - RecallActions: []string{*recallActions}, - RecallPrefix: *recallPrefix, + RecallActionsRaw: *recallActions, + RecallPrefix: *recallPrefix, + RecallStrategy: *recallStrategy, }, } if *recall { diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index e5fc2b1..a0ff6df 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -31,7 +31,7 @@ func main() { dir := usr.HomeDir pidfilePath := filepath.Join(dir, ".resh/resh.pid") configPath := filepath.Join(dir, ".config/resh.toml") - outputPath := filepath.Join(dir, ".resh_history.json") + historyPath := filepath.Join(dir, ".resh_history.json") logPath := filepath.Join(dir, ".resh/daemon.log") f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) @@ -69,7 +69,7 @@ func main() { if err != nil { log.Fatal("Could not create pidfile", err) } - runServer(config, outputPath) + runServer(config, historyPath) err = os.Remove(pidfilePath) if err != nil { log.Println("Could not delete pidfile", err) diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index 92d725f..6eb44b7 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -11,7 +11,7 @@ import ( "github.com/curusarn/resh/pkg/sesswatch" ) -func runServer(config cfg.Config, outputPath string) { +func runServer(config cfg.Config, historyPath string) { var recordSubscribers []chan records.Record var sessionInitSubscribers []chan records.Record var sessionDropSubscribers []chan string @@ -23,14 +23,16 @@ func runServer(config cfg.Config, outputPath string) { sessionDropSubscribers = append(sessionDropSubscribers, sesshistSessionsToDrop) sesshistRecords := make(chan records.Record) recordSubscribers = append(recordSubscribers, sesshistRecords) - sesshistDispatch := sesshist.NewDispatch(sesshistSessionsToInit, sesshistSessionsToDrop, sesshistRecords) // histfile histfileRecords := make(chan records.Record) recordSubscribers = append(recordSubscribers, histfileRecords) histfileSessionsToDrop := make(chan string) sessionDropSubscribers = append(sessionDropSubscribers, histfileSessionsToDrop) - histfile.Go(histfileRecords, outputPath, histfileSessionsToDrop) + histfileBox := histfile.New(histfileRecords, historyPath, 10000, histfileSessionsToDrop) + + // sesshist New + sesshistDispatch := sesshist.NewDispatch(sesshistSessionsToInit, sesshistSessionsToDrop, sesshistRecords, histfileBox, config.SesshistInitHistorySize) // sesswatch sesswatchSessionsToWatch := make(chan records.Record) diff --git a/conf/config.toml b/conf/config.toml index 6642be0..1970f34 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -1,2 +1,3 @@ port = 2627 sesswatchPeriodSeconds = 120 +sesshistInitHistorySize = 1000 diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index 0e5fb61..16726bb 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -2,6 +2,7 @@ package cfg // Config struct type Config struct { - Port int - SesswatchPeriodSeconds uint + Port int + SesswatchPeriodSeconds uint + SesshistInitHistorySize int } diff --git a/pkg/histfile/histfile.go b/pkg/histfile/histfile.go index 5418c45..879efe1 100644 --- a/pkg/histfile/histfile.go +++ b/pkg/histfile/histfile.go @@ -9,31 +9,49 @@ import ( "github.com/curusarn/resh/pkg/records" ) -type histfile struct { - mutex sync.Mutex - sessions map[string]records.Record - outputPath string +// Histfile writes records to histfile +type Histfile struct { + sessionsMutex sync.Mutex + sessions map[string]records.Record + historyPath string + + recentMutex sync.Mutex + recentRecords []records.Record + recentCmdLines []string // deduplicated + cmdLinesLastIndex map[string]int } -// Go creates histfile and runs two gorutines on it -func Go(input chan records.Record, outputPath string, sessionsToDrop chan string) { - hf := histfile{sessions: map[string]records.Record{}, outputPath: outputPath} +// New creates new histfile and runs two gorutines on it +func New(input chan records.Record, historyPath string, initHistSize int, sessionsToDrop chan string) *Histfile { + hf := Histfile{ + sessions: map[string]records.Record{}, + historyPath: historyPath, + cmdLinesLastIndex: map[string]int{}, + } + go hf.loadHistory(initHistSize) go hf.writer(input) go hf.sessionGC(sessionsToDrop) + return &hf +} + +func (h *Histfile) loadHistory(initHistSize int) { + h.recentMutex.Lock() + defer h.recentMutex.Unlock() + h.recentCmdLines = records.LoadCmdLinesFromFile(h.historyPath, initHistSize) } // sessionGC reads sessionIDs from channel and deletes them from histfile struct -func (h *histfile) sessionGC(sessionsToDrop chan string) { +func (h *Histfile) sessionGC(sessionsToDrop chan string) { for { func() { session := <-sessionsToDrop log.Println("histfile: got session to drop", session) - h.mutex.Lock() - defer h.mutex.Unlock() + h.sessionsMutex.Lock() + defer h.sessionsMutex.Unlock() if part1, found := h.sessions[session]; found == true { log.Println("histfile: Dropping session:", session) delete(h.sessions, session) - go writeRecord(part1, h.outputPath) + go writeRecord(part1, h.historyPath) } else { log.Println("histfile: No hanging parts for session:", session) } @@ -42,38 +60,52 @@ func (h *histfile) sessionGC(sessionsToDrop chan string) { } // writer reads records from channel, merges them and writes them to file -func (h *histfile) writer(input chan records.Record) { +func (h *Histfile) writer(input chan records.Record) { for { func() { record := <-input - h.mutex.Lock() - defer h.mutex.Unlock() + h.sessionsMutex.Lock() + defer h.sessionsMutex.Unlock() if record.PartOne { if _, found := h.sessions[record.SessionID]; found { - log.Println("histfile ERROR: Got another first part of the records before merging the previous one - overwriting!") + log.Println("histfile WARN: Got another first part of the records before merging the previous one - overwriting! " + + "(this happens in bash because bash-preexec runs when it's not supposed to)") } h.sessions[record.SessionID] = record } else { - part1, found := h.sessions[record.SessionID] - if found == false { + if part1, found := h.sessions[record.SessionID]; found == false { log.Println("histfile ERROR: Got second part of records and nothing to merge it with - ignoring!") } else { delete(h.sessions, record.SessionID) - go mergeAndWriteRecord(part1, record, h.outputPath) + go h.mergeAndWriteRecord(part1, record) } } }() } } -func mergeAndWriteRecord(part1, part2 records.Record, outputPath string) { +func (h *Histfile) mergeAndWriteRecord(part1, part2 records.Record) { err := part1.Merge(part2) if err != nil { log.Println("Error while merging", err) return } - writeRecord(part1, outputPath) + + func() { + h.recentMutex.Lock() + defer h.recentMutex.Unlock() + h.recentRecords = append(h.recentRecords, part1) + cmdLine := part1.CmdLine + idx, found := h.cmdLinesLastIndex[cmdLine] + if found { + h.recentCmdLines = append(h.recentCmdLines[:idx], h.recentCmdLines[idx+1:]...) + } + h.cmdLinesLastIndex[cmdLine] = len(h.recentCmdLines) + h.recentCmdLines = append(h.recentCmdLines, cmdLine) + }() + + writeRecord(part1, h.historyPath) } func writeRecord(rec records.Record, outputPath string) { @@ -95,3 +127,8 @@ func writeRecord(rec records.Record, outputPath string) { return } } + +// GetRecentCmdLines returns recent cmdLines +func (h *Histfile) GetRecentCmdLines(limit int) []string { + return h.recentCmdLines +} diff --git a/pkg/records/records.go b/pkg/records/records.go index d3668a5..3b4170a 100644 --- a/pkg/records/records.go +++ b/pkg/records/records.go @@ -1,10 +1,12 @@ package records import ( + "bufio" "encoding/json" "errors" "log" "math" + "os" "strconv" "strings" @@ -81,10 +83,11 @@ type BaseRecord struct { SessionExit bool `json:"sessionExit,omitempty"` // recall metadata - Recalled bool `json:"recalled"` - RecallHistno int `json:"recallHistno,omitempty"` - RecallStrategy string `json:"recallStrategy,omitempty"` - RecallActions []string `json:"recallActions,omitempty"` + Recalled bool `json:"recalled"` + RecallHistno int `json:"recallHistno,omitempty"` + RecallStrategy string `json:"recallStrategy,omitempty"` + RecallActionsRaw string `json:"recallActionsRaw,omitempty"` + RecallActions []string `json:"recallActions,omitempty"` // recall command RecallPrefix string `json:"recallPrefix,omitempty"` @@ -177,11 +180,10 @@ func (r *Record) Merge(r2 Record) error { if r.SessionID != r2.SessionID { return errors.New("Records to merge are not from the same sesion - r1:" + r.SessionID + " r2:" + r2.SessionID) } - if r.CmdLine != r2.CmdLine || r.RealtimeBefore != r2.RealtimeBefore { - return errors.New("Records to merge are not parts of the same records - r1:" + - r.CmdLine + "(" + strconv.FormatFloat(r.RealtimeBefore, 'f', -1, 64) + ") r2:" + - r2.CmdLine + "(" + strconv.FormatFloat(r2.RealtimeBefore, 'f', -1, 64) + ")") + if r.CmdLine != r2.CmdLine { + return errors.New("Records to merge are not parts of the same records - r1:" + r.CmdLine + " r2:" + r2.CmdLine) } + // r.RealtimeBefore != r2.RealtimeBefore - can't be used because of bash-preexec runs when it's not supposed to r.ExitCode = r2.ExitCode r.PwdAfter = r2.PwdAfter r.RealPwdAfter = r2.RealPwdAfter @@ -436,3 +438,50 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { return dist } + +// LoadCmdLinesFromFile loads limit cmdlines from file +func LoadCmdLinesFromFile(fname string, limit int) []string { + recs := LoadFromFile(fname, limit*3) // assume that at least 1/3 of commands is unique + var cmdLines []string + cmdLinesSet := map[string]bool{} + for i := len(recs) - 1; i >= 0; i-- { + cmdLine := recs[i].CmdLine + if cmdLinesSet[cmdLine] { + continue + } + cmdLinesSet[cmdLine] = true + cmdLines = append([]string{cmdLine}, cmdLines...) + if len(cmdLines) > limit { + break + } + } + return cmdLines +} + +// LoadFromFile loads at most 'limit' records from 'fname' file +func LoadFromFile(fname string, limit int) []Record { + file, err := os.Open(fname) + if err != nil { + log.Fatal("Open() resh history file error:", err) + } + defer file.Close() + + var recs []Record + scanner := bufio.NewScanner(file) + for scanner.Scan() { + record := Record{} + fallbackRecord := FallbackRecord{} + line := scanner.Text() + err = json.Unmarshal([]byte(line), &record) + if err != nil { + err = json.Unmarshal([]byte(line), &fallbackRecord) + if err != nil { + log.Println("Line:", line) + log.Fatal("Decoding error:", err) + } + record = Convert(&fallbackRecord) + } + recs = append(recs, record) + } + return recs +} diff --git a/pkg/sesshist/sesshist.go b/pkg/sesshist/sesshist.go index 09f7e69..50b79ec 100644 --- a/pkg/sesshist/sesshist.go +++ b/pkg/sesshist/sesshist.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/curusarn/resh/pkg/histfile" "github.com/curusarn/resh/pkg/records" ) @@ -14,11 +15,20 @@ import ( type Dispatch struct { sessions map[string]*sesshist mutex sync.RWMutex + + history *histfile.Histfile + historyInitSize int } // NewDispatch creates a new sesshist.Dispatch and starts necessary gorutines -func NewDispatch(sessionsToInit chan records.Record, sessionsToDrop chan string, recordsToAdd chan records.Record) *Dispatch { - s := Dispatch{sessions: map[string]*sesshist{}} +func NewDispatch(sessionsToInit chan records.Record, sessionsToDrop chan string, + recordsToAdd chan records.Record, history *histfile.Histfile, historyInitSize int) *Dispatch { + + s := Dispatch{ + sessions: map[string]*sesshist{}, + history: history, + historyInitSize: historyInitSize, + } go s.sessionInitializer(sessionsToInit) go s.sessionDropper(sessionsToDrop) go s.recordAdder(recordsToAdd) @@ -54,6 +64,7 @@ func (s *Dispatch) recordAdder(recordsToAdd chan records.Record) { // InitSession struct func (s *Dispatch) initSession(sessionID string) error { + log.Println("sesshist: initializing session - " + sessionID) s.mutex.RLock() _, found := s.sessions[sessionID] s.mutex.RUnlock() @@ -62,9 +73,17 @@ func (s *Dispatch) initSession(sessionID string) error { return errors.New("sesshist ERROR: Can't INIT already existing session " + sessionID) } + log.Println("sesshist: loading history to populate session - " + sessionID) + historyCmdLines := s.history.GetRecentCmdLines(s.historyInitSize) + s.mutex.Lock() defer s.mutex.Unlock() - s.sessions[sessionID] = &sesshist{} + // init sesshist and populate it with history loaded from file + s.sessions[sessionID] = &sesshist{ + recentCmdLines: historyCmdLines, + cmdLinesLastIndex: map[string]int{}, + } + log.Println("sesshist: session init done - " + sessionID) return nil } @@ -91,19 +110,22 @@ func (s *Dispatch) addRecentRecord(sessionID string, record records.Record) erro s.mutex.RUnlock() if found == false { - return errors.New("sesshist ERROR: No session history for SessionID " + sessionID + " (should we create one?)") + log.Println("sesshist ERROR: addRecontRecord(): No session history for SessionID " + sessionID + " - creating session history.") + s.initSession(sessionID) + return s.addRecentRecord(sessionID, record) } session.mutex.Lock() defer session.mutex.Unlock() session.recentRecords = append(session.recentRecords, record) // remove previous occurance of record - for i := len(session.recentCmdLines) - 1; i >= 0; i-- { - if session.recentCmdLines[i] == record.CmdLine { - session.recentCmdLines = append(session.recentCmdLines[:i], session.recentCmdLines[i+1:]...) - } + cmdLine := record.CmdLine + idx, found := session.cmdLinesLastIndex[cmdLine] + if found { + session.recentCmdLines = append(session.recentCmdLines[:idx], session.recentCmdLines[idx+1:]...) } + session.cmdLinesLastIndex[cmdLine] = len(session.recentCmdLines) // append new record - session.recentCmdLines = append(session.recentCmdLines, record.CmdLine) + session.recentCmdLines = append(session.recentCmdLines, cmdLine) log.Println("sesshist: record:", record.CmdLine, "; added to session:", sessionID, "; session len:", len(session.recentCmdLines), "; session len w/ dups:", len(session.recentRecords)) return nil @@ -116,7 +138,8 @@ func (s *Dispatch) Recall(sessionID string, histno int, prefix string) (string, s.mutex.RUnlock() if found == false { - return "", errors.New("sesshist ERROR: No session history for SessionID " + sessionID) + // go s.initSession(sessionID) + return "", errors.New("sesshist ERROR: No session history for SessionID " + sessionID + " - should we create one?") } if prefix == "" { session.mutex.Lock() @@ -129,10 +152,10 @@ func (s *Dispatch) Recall(sessionID string, histno int, prefix string) (string, } type sesshist struct { - recentRecords []records.Record - recentCmdLines []string // deduplicated - // cmdLines map[string]int - mutex sync.Mutex + recentRecords []records.Record + recentCmdLines []string // deduplicated + cmdLinesLastIndex map[string]int + mutex sync.Mutex } func (s *sesshist) getRecordByHistno(histno int) (string, error) { diff --git a/scripts/hooks.sh b/scripts/hooks.sh index be11652..1595a64 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -5,16 +5,17 @@ __resh_reset_variables() { __RESH_HIST_PREV_LINE="" __RESH_HIST_RECALL_ACTIONS="" __RESH_HIST_NO_PREFIX_MODE=0 + __RESH_HIST_RECALL_STRATEGY="" } __resh_preexec() { # core __RESH_COLLECT=1 __RESH_CMDLINE="$1" # not local to preserve it for postcollect (useful as sanity check) - __resh_collect --cmdLine "$__RESH_CMDLINE" --recall-actions "$__RESH_HIST_RECALL_ACTIONS" \ + __resh_collect --cmdLine "$__RESH_CMDLINE" \ + --recall-actions "$__RESH_HIST_RECALL_ACTIONS" \ + --recall-strategy "$__RESH_HIST_RECALL_STRATEGY" \ &>~/.resh/collect_last_run_out.txt || echo "resh-collect ERROR: $(head -n 1 ~/.resh/collect_last_run_out.txt)" - - __resh_reset_variables } # used for collect and collect --recall @@ -152,6 +153,7 @@ __resh_precmd() { -timezoneAfter "$__RESH_TZ_AFTER" \ &>~/.resh/postcollect_last_run_out.txt || echo "resh-postcollect ERROR: $(head -n 1 ~/.resh/postcollect_last_run_out.txt)" fi + __resh_reset_variables fi unset __RESH_COLLECT } diff --git a/scripts/reshctl.sh b/scripts/reshctl.sh index 1f59217..2656055 100644 --- a/scripts/reshctl.sh +++ b/scripts/reshctl.sh @@ -5,8 +5,8 @@ . ~/.resh/widgets.sh __resh_bind_arrows() { - bindfunc '\C-k' __resh_widget_arrow_up_compat - bindfunc '\C-j' __resh_widget_arrow_down_compat + bindfunc '\e[A' __resh_widget_arrow_up_compat + bindfunc '\e[B' __resh_widget_arrow_down_compat return 0 } diff --git a/scripts/widgets.sh b/scripts/widgets.sh index 9960da7..d1d48e5 100644 --- a/scripts/widgets.sh +++ b/scripts/widgets.sh @@ -3,16 +3,21 @@ . ~/.resh/hooks.sh __resh_helper_arrow_pre() { + # this is a very bad workaround + # force bash-preexec to run repeatedly because otherwise premature run of bash-preexec overshadows the next poper run + # I honestly think that it's impossible to make widgets work in bash without hacks like this + # shellcheck disable=2034 + __bp_preexec_interactive_mode="on" + # set recall strategy + __RESH_HIST_RECALL_STRATEGY="bash_recent - history-search-{backward,forward}" # set prefix - __RESH_PREFIX=${BUFFER:0:CURSOR} + __RESH_PREFIX=${BUFFER:0:$CURSOR} # cursor not at the end of the line => end "NO_PREFIX_MODE" [ "$CURSOR" -ne "${#BUFFER}" ] && __RESH_HIST_NO_PREFIX_MODE=0 # if user made any edits from last recall action => restart histno AND deactivate "NO_PREFIX_MODE" [ "$BUFFER" != "$__RESH_HIST_PREV_LINE" ] && __RESH_HISTNO=0 && __RESH_HIST_NO_PREFIX_MODE=0 # "NO_PREFIX_MODE" => set prefix to empty string [ "$__RESH_HIST_NO_PREFIX_MODE" -eq 1 ] && __RESH_PREFIX="" - # append curent recall action - __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;arrow_up:$__RESH_PREFIX" # histno == 0 => save current line [ $__RESH_HISTNO -eq 0 ] && __RESH_HISTNO_ZERO_LINE=$BUFFER } @@ -28,18 +33,20 @@ __resh_helper_arrow_post() { __resh_widget_arrow_up() { # run helper function __resh_helper_arrow_pre + # append curent recall action + __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;arrow_up:$__RESH_PREFIX" # increment histno - (( __RESH_HISTNO++ )) + __RESH_HISTNO=$((__RESH_HISTNO+1)) # back at histno == 0 => restore original line if [ $__RESH_HISTNO -eq 0 ]; then BUFFER=$__RESH_HISTNO_ZERO_LINE else # run recall local NEW_BUFFER - NEW_BUFFER="$(__resh_collect --recall --histno "$__RESH_HISTNO" --prefix-search "$__RESH_PREFIX" 2> ~/.resh/arrow_up_last_run_out.txt)" + NEW_BUFFER="$(__resh_collect --recall --prefix-search "$__RESH_PREFIX" 2> ~/.resh/arrow_up_last_run_out.txt)" # IF new buffer in non-empty THEN use the new buffer ELSE revert histno change # shellcheck disable=SC2015 - [ ${#NEW_BUFFER} -gt 0 ] && BUFFER=$NEW_BUFFER || (( __RESH_HISTNO-- )) + [ ${#NEW_BUFFER} -gt 0 ] && BUFFER=$NEW_BUFFER || __RESH_HISTNO=$((__RESH_HISTNO-1)) fi # run post helper __resh_helper_arrow_post @@ -47,8 +54,10 @@ __resh_widget_arrow_up() { __resh_widget_arrow_down() { # run helper function __resh_helper_arrow_pre + # append curent recall action + __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;arrow_down:$__RESH_PREFIX" # increment histno - (( __RESH_HISTNO-- )) + __RESH_HISTNO=$((__RESH_HISTNO-1)) # prevent HISTNO from getting negative (for now) [ $__RESH_HISTNO -lt 0 ] && __RESH_HISTNO=0 # back at histno == 0 => restore original line @@ -57,7 +66,7 @@ __resh_widget_arrow_down() { else # run recall local NEW_BUFFER - NEW_BUFFER="$(__resh_collect --recall --histno "$__RESH_HISTNO" --prefix-search "$__RESH_PREFIX" 2> ~/.resh/arrow_down_last_run_out.txt)" + NEW_BUFFER="$(__resh_collect --recall --prefix-search "$__RESH_PREFIX" 2> ~/.resh/arrow_down_last_run_out.txt)" # IF new buffer in non-empty THEN use the new buffer ELSE revert histno change # shellcheck disable=SC2015 [ ${#NEW_BUFFER} -gt 0 ] && BUFFER=$NEW_BUFFER || (( __RESH_HISTNO++ )) @@ -65,8 +74,8 @@ __resh_widget_arrow_down() { __resh_helper_arrow_post } __resh_widget_control_R() { - local __RESH_LBUFFER=${BUFFER:0:CURSOR} - __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;control_R:$__RESH_LBUFFER" + local __RESH_PREFIX=${BUFFER:0:CURSOR} + __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;control_R:$__RESH_PREFIX" # resh-collect --hstr hstr }