Mirror
/
resh
mirror of https://github.com/curusarn/resh
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

load bash/zsh history when resh hostory is too small, fix duplicates in sesshist, fix lag at the end of hist session

Browse Source
pull/30/head
Simon Let 6 years ago
parent 15a431cfc3
commit aba197561a
8 changed files with 210 additions and 57 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      cmd/daemon/main.go
  2. 9
      cmd/daemon/run-server.go
  3. 85
      pkg/histfile/histfile.go
  4. 37
      pkg/histlist/histlist.go
  5. 88
      pkg/records/records.go
  6. 26
      pkg/sesshist/sesshist.go
  7. 1
      scripts/hooks.sh
  8. 15
      scripts/widgets.sh

6
cmd/daemon/main.go
Unescape View File

@ -31,7 +31,9 @@ func main() {
dir := usr.HomeDir
pidfilePath := filepath.Join(dir, ".resh/resh.pid")
configPath := filepath.Join(dir, ".config/resh.toml")
historyPath := filepath.Join(dir, ".resh_history.json")
reshHistoryPath := filepath.Join(dir, ".resh_history.json")
bashHistoryPath := filepath.Join(dir, ".bash_history")
zshHistoryPath := filepath.Join(dir, ".zsh_history")
logPath := filepath.Join(dir, ".resh/daemon.log")
f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
@ -73,7 +75,7 @@ func main() {
if err != nil {
log.Fatal("Could not create pidfile", err)
}
runServer(config, historyPath)
runServer(config, reshHistoryPath, bashHistoryPath, zshHistoryPath)
log.Println("main: Removing pidfile ...")
err = os.Remove(pidfilePath)
if err != nil {

9
cmd/daemon/run-server.go
Unescape View File

@ -13,7 +13,7 @@ import (
"github.com/curusarn/resh/pkg/signalhandler"
)
func runServer(config cfg.Config, historyPath string) {
func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPath string) {
var recordSubscribers []chan records.Record
var sessionInitSubscribers []chan records.Record
var sessionDropSubscribers []chan string
@ -36,7 +36,12 @@ func runServer(config cfg.Config, historyPath string) {
sessionDropSubscribers = append(sessionDropSubscribers, histfileSessionsToDrop)
histfileSignals := make(chan os.Signal)
signalSubscribers = append(signalSubscribers, histfileSignals)
histfileBox := histfile.New(histfileRecords, historyPath, 10000, histfileSessionsToDrop, histfileSignals, shutdown)
maxHistSize := 10000 // lines
minHistSizeKB := 100 // roughly lines
histfileBox := histfile.New(histfileRecords, histfileSessionsToDrop,
reshHistoryPath, bashHistoryPath, zshHistoryPath,
maxHistSize, minHistSizeKB,
histfileSignals, shutdown)
// sesshist New
sesshistDispatch := sesshist.NewDispatch(sesshistSessionsToInit, sesshistSessionsToDrop,

85
pkg/histfile/histfile.go
Unescape View File

@ -3,6 +3,7 @@ package histfile
import (
"encoding/json"
"log"
"math"
"os"
"strconv"
"sync"
@ -20,30 +21,68 @@ type Histfile struct {
recentMutex sync.Mutex
recentRecords []records.Record
cmdLines histlist.Histlist
// NOTE: we have separate histories which only differ if there was not enough resh_history
// resh_history itself is common for both bash and zsh
bashCmdLines histlist.Histlist
zshCmdLines histlist.Histlist
}
// New creates new histfile and runs two gorutines on it
func New(input chan records.Record, historyPath string, initHistSize int, sessionsToDrop chan string,
// New creates new histfile and runs its gorutines
func New(input chan records.Record, sessionsToDrop chan string,
reshHistoryPath string, bashHistoryPath string, zshHistoryPath string,
maxInitHistSize int, minInitHistSizeKB int,
signals chan os.Signal, shutdownDone chan string) *Histfile {
hf := Histfile{
sessions: map[string]records.Record{},
historyPath: historyPath,
cmdLines: histlist.New(),
sessions: map[string]records.Record{},
historyPath: reshHistoryPath,
bashCmdLines: histlist.New(),
zshCmdLines: histlist.New(),
}
go hf.loadHistory(initHistSize)
go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB)
go hf.writer(input, signals, shutdownDone)
go hf.sessionGC(sessionsToDrop)
return &hf
}
func (h *Histfile) loadHistory(initHistSize int) {
// loadsHistory from resh_history and if there is not enough of it also load native shell histories
func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHistSize, minInitHistSizeKB int) {
h.recentMutex.Lock()
defer h.recentMutex.Unlock()
log.Println("histfile: Loading history from file ...")
h.cmdLines = records.LoadCmdLinesFromFile(h.historyPath, initHistSize)
log.Println("histfile: History loaded - cmdLine count:", len(h.cmdLines.List))
log.Println("histfile: Checking if resh_history is large enough ...")
fi, err := os.Stat(h.historyPath)
var size int
if err != nil {
log.Println("histfile ERROR: failed to stat resh_history file:", err)
} else {
size = int(fi.Size())
}
useNativeHistories := false
if size/1024 < minInitHistSizeKB {
useNativeHistories = true
log.Println("histfile WARN: resh_history is too small - loading native bash and zsh history ...")
h.bashCmdLines = records.LoadCmdLinesFromBashFile(bashHistoryPath)
log.Println("histfile: bash history loaded - cmdLine count:", len(h.bashCmdLines.List))
h.zshCmdLines = records.LoadCmdLinesFromZshFile(zshHistoryPath)
log.Println("histfile: zsh history loaded - cmdLine count:", len(h.zshCmdLines.List))
// no maxInitHistSize when using native histories
maxInitHistSize = math.MaxInt32
}
log.Println("histfile: Loading resh history from file ...")
reshCmdLines := histlist.New()
// NOTE: keeping this weird interface for now because we might use it in the future
// when we only load bash or zsh history
records.LoadCmdLinesFromFile(&reshCmdLines, h.historyPath, maxInitHistSize)
log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List))
if useNativeHistories == false {
h.bashCmdLines = reshCmdLines
h.zshCmdLines = histlist.Copy(reshCmdLines)
return
}
h.bashCmdLines.AddHistlist(reshCmdLines)
log.Println("histfile: bash history + resh history - cmdLine count:", len(h.bashCmdLines.List))
h.zshCmdLines.AddHistlist(reshCmdLines)
log.Println("histfile: zsh history + resh history - cmdLine count:", len(h.zshCmdLines.List))
}
// sessionGC reads sessionIDs from channel and deletes them from histfile struct
@ -124,12 +163,8 @@ func (h *Histfile) mergeAndWriteRecord(part1, part2 records.Record) {
defer h.recentMutex.Unlock()
h.recentRecords = append(h.recentRecords, part1)
cmdLine := part1.CmdLine
idx, found := h.cmdLines.LastIndex[cmdLine]
if found {
h.cmdLines.List = append(h.cmdLines.List[:idx], h.cmdLines.List[idx+1:]...)
}
h.cmdLines.LastIndex[cmdLine] = len(h.cmdLines.List)
h.cmdLines.List = append(h.cmdLines.List, cmdLine)
h.bashCmdLines.AddCmdLine(cmdLine)
h.zshCmdLines.AddCmdLine(cmdLine)
}()
writeRecord(part1, h.historyPath)
@ -156,11 +191,21 @@ func writeRecord(rec records.Record, outputPath string) {
}
// GetRecentCmdLines returns recent cmdLines
func (h *Histfile) GetRecentCmdLines(limit int) histlist.Histlist {
func (h *Histfile) GetRecentCmdLines(shell string, limit int) histlist.Histlist {
// NOTE: limit does nothing atm
h.recentMutex.Lock()
defer h.recentMutex.Unlock()
log.Println("histfile: History requested ...")
hl := histlist.Copy(h.cmdLines)
log.Println("histfile: History copied - cmdLine count:", len(hl.List))
var hl histlist.Histlist
if shell == "bash" {
hl = histlist.Copy(h.bashCmdLines)
log.Println("histfile: history copied (bash) - cmdLine count:", len(hl.List))
return hl
}
if shell != "zsh" {
log.Println("histfile ERROR: Unknown shell: ", shell)
}
hl = histlist.Copy(h.zshCmdLines)
log.Println("histfile: history copied (zsh) - cmdLine count:", len(hl.List))
return hl
}

37
pkg/histlist/histlist.go
Unescape View File

@ -1,5 +1,7 @@
package histlist
import "log"
// Histlist is a deduplicated list of cmdLines
type Histlist struct {
// list of commands lines (deduplicated)
@ -25,3 +27,38 @@ func Copy(hl Histlist) Histlist {
}
return newHl
}
// AddCmdLine to the histlist
func (h *Histlist) AddCmdLine(cmdLine string) {
lenBefore := len(h.List)
// lookup
idx, found := h.LastIndex[cmdLine]
if found {
// remove duplicate
if cmdLine != h.List[idx] {
log.Println("histlist ERROR: Adding cmdLine:", cmdLine, " != LastIndex[cmdLine]:", h.List[idx])
}
h.List = append(h.List[:idx], h.List[idx+1:]...)
// idx++
for idx < len(h.List) {
cmdLn := h.List[idx]
h.LastIndex[cmdLn]--
if idx != h.LastIndex[cmdLn] {
log.Println("histlist ERROR: Shifting LastIndex idx:", idx, " != LastIndex[cmdLn]:", h.LastIndex[cmdLn])
}
idx++
}
}
// update last index
h.LastIndex[cmdLine] = len(h.List)
// append new cmdline
h.List = append(h.List, cmdLine)
log.Println("histlist: Added cmdLine:", cmdLine, "; history length:", lenBefore, "->", len(h.List))
}
// AddHistlist contents of another histlist to this histlist
func (h *Histlist) AddHistlist(h2 Histlist) {
for _, cmdLine := range h2.List {
h.AddCmdLine(cmdLine)
}
}

88
pkg/records/records.go
Unescape View File

@ -455,9 +455,10 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 {
return dist
}
// LoadCmdLinesFromFile loads limit cmdlines from file
func LoadCmdLinesFromFile(fname string, limit int) histlist.Histlist {
// LoadCmdLinesFromFile loads cmdlines from file
func LoadCmdLinesFromFile(hl *histlist.Histlist, fname string, limit int) {
recs := LoadFromFile(fname, limit*3) // assume that at least 1/3 of commands is unique
// go from bottom and deduplicate
var cmdLines []string
cmdLinesSet := map[string]bool{}
for i := len(recs) - 1; i >= 0; i-- {
@ -471,24 +472,24 @@ func LoadCmdLinesFromFile(fname string, limit int) histlist.Histlist {
break
}
}
hl := histlist.New()
hl.List = cmdLines
for idx, cmdLine := range cmdLines {
hl.LastIndex[cmdLine] = idx
// add everything to histlist
for _, cmdLine := range cmdLines {
hl.AddCmdLine(cmdLine)
}
return hl
}
// LoadFromFile loads at most 'limit' records from 'fname' file
// LoadFromFile loads records from 'fname' file
func LoadFromFile(fname string, limit int) []Record {
// NOTE: limit does nothing atm
var recs []Record
file, err := os.Open(fname)
if err != nil {
log.Fatal("Open() resh history file error:", err)
log.Println("Open() resh history file error:", err)
log.Println("WARN: Skipping reading resh history!")
return recs
}
defer file.Close()
var recs []Record
scanner := bufio.NewScanner(file)
for scanner.Scan() {
record := Record{}
@ -507,3 +508,70 @@ func LoadFromFile(fname string, limit int) []Record {
}
return recs
}
// LoadCmdLinesFromZshFile loads cmdlines from zsh history file
func LoadCmdLinesFromZshFile(fname string) histlist.Histlist {
file, err := os.Open(fname)
if err != nil {
log.Fatal("Open() resh history file error:", err)
}
defer file.Close()
hl := histlist.New()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
// trim newline
line = strings.TrimRight(line, "\n")
var cmd string
// zsh format EXTENDED_HISTORY
// : 1576270617:0;make install
// zsh format no EXTENDED_HISTORY
// make install
if len(line) == 0 {
// skip empty
continue
}
if strings.Contains(line, ":") && strings.Contains(line, ";") &&
len(strings.Split(line, ":")) >= 3 && len(strings.Split(line, ";")) >= 2 {
// contains at least 2x ':' and 1x ';' => assume EXTENDED_HISTORY
cmd = strings.Split(line, ";")[1]
} else {
cmd = line
}
hl.AddCmdLine(cmd)
}
return hl
}
// LoadCmdLinesFromBashFile loads cmdlines from bash history file
func LoadCmdLinesFromBashFile(fname string) histlist.Histlist {
file, err := os.Open(fname)
if err != nil {
log.Fatal("Open() resh history file error:", err)
}
defer file.Close()
hl := histlist.New()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
// trim newline
line = strings.TrimRight(line, "\n")
// trim spaces from left
line = strings.TrimLeft(line, " ")
// bash format (two lines)
// #1576199174
// make install
if strings.HasPrefix(line, "#") {
// is either timestamp or comment => skip
continue
}
if len(line) == 0 {
// skip empty
continue
}
hl.AddCmdLine(line)
}
return hl
}

26
pkg/sesshist/sesshist.go
Unescape View File

@ -40,7 +40,7 @@ func (s *Dispatch) sessionInitializer(sessionsToInit chan records.Record) {
for {
record := <-sessionsToInit
log.Println("sesshist: got session to init - " + record.SessionID)
s.initSession(record.SessionID)
s.initSession(record.SessionID, record.Shell)
}
}
@ -64,7 +64,7 @@ func (s *Dispatch) recordAdder(recordsToAdd chan records.Record) {
}
// InitSession struct
func (s *Dispatch) initSession(sessionID string) error {
func (s *Dispatch) initSession(sessionID string, shell string) error {
log.Println("sesshist: initializing session - " + sessionID)
s.mutex.RLock()
_, found := s.sessions[sessionID]
@ -75,7 +75,7 @@ func (s *Dispatch) initSession(sessionID string) error {
}
log.Println("sesshist: loading history to populate session - " + sessionID)
historyCmdLines := s.history.GetRecentCmdLines(s.historyInitSize)
historyCmdLines := s.history.GetRecentCmdLines(shell, s.historyInitSize)
s.mutex.Lock()
defer s.mutex.Unlock()
@ -113,29 +113,15 @@ func (s *Dispatch) addRecentRecord(sessionID string, record records.Record) erro
s.mutex.RUnlock()
if found == false {
log.Println("sesshist ERROR: addRecontRecord(): No session history for SessionID " + sessionID + " - creating session history.")
s.initSession(sessionID)
log.Println("sesshist ERROR: addRecentRecord(): No session history for SessionID " + sessionID + " - creating session history.")
s.initSession(sessionID, record.Shell)
return s.addRecentRecord(sessionID, record)
}
log.Println("sesshist: RLocking session lock (w/ defer) ...")
session.mutex.Lock()
defer session.mutex.Unlock()
session.recentRecords = append(session.recentRecords, record)
// remove previous occurance of record
log.Println("sesshist: Looking for duplicate cmdLine ...")
cmdLine := record.CmdLine
// trim spaces to have less duplicates in the sesshist
cmdLine = strings.TrimRight(cmdLine, " ")
idx, found := session.recentCmdLines.LastIndex[cmdLine]
if found {
log.Println("sesshist: Removing duplicate cmdLine at index:", idx, " out of", len(session.recentCmdLines.List), "...")
session.recentCmdLines.List = append(session.recentCmdLines.List[:idx], session.recentCmdLines.List[idx+1:]...)
}
log.Println("sesshist: Updating last index ...")
session.recentCmdLines.LastIndex[cmdLine] = len(session.recentCmdLines.List)
// append new record
log.Println("sesshist: Appending cmdLine ...")
session.recentCmdLines.List = append(session.recentCmdLines.List, cmdLine)
session.recentCmdLines.AddCmdLine(record.CmdLine)
log.Println("sesshist: record:", record.CmdLine, "; added to session:", sessionID,
"; session len:", len(session.recentCmdLines.List), "; session len (records):", len(session.recentRecords))
return nil

1
scripts/hooks.sh
Unescape View File

@ -1,6 +1,7 @@
__resh_reset_variables() {
__RESH_HISTNO=0
__RESH_HISTNO_MAX=""
__RESH_HISTNO_ZERO_LINE=""
__RESH_HIST_PREV_LINE=""
__RESH_HIST_RECALL_ACTIONS=""

15
scripts/widgets.sh
Unescape View File

@ -37,8 +37,12 @@ __resh_widget_arrow_up() {
__RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;arrow_up:$__RESH_PREFIX"
# increment histno
__RESH_HISTNO=$((__RESH_HISTNO+1))
# back at histno == 0 => restore original line
if [ "$__RESH_HISTNO" -eq 0 ]; then
if [ "${#__RESH_HISTNO_MAX}" -gt 0 ] && [ "${__RESH_HISTNO}" -gt "${__RESH_HISTNO_MAX}" ]; then
# end of the session -> don't recall, do nothing
# fix histno
__RESH_HISTNO=$((__RESH_HISTNO-1))
elif [ "$__RESH_HISTNO" -eq 0 ]; then
# back at histno == 0 => restore original line
BUFFER=$__RESH_HISTNO_ZERO_LINE
else
# run recall
@ -46,7 +50,12 @@ __resh_widget_arrow_up() {
NEW_BUFFER="$(__resh_collect --recall --prefix-search "$__RESH_PREFIX" 2> ~/.resh/arrow_up_last_run_out.txt)"
# IF new buffer in non-empty THEN use the new buffer ELSE revert histno change
# shellcheck disable=SC2015
[ "${#NEW_BUFFER}" -gt 0 ] && BUFFER=$NEW_BUFFER || __RESH_HISTNO=$((__RESH_HISTNO-1))
if [ "${#NEW_BUFFER}" -gt 0 ]; then
BUFFER=$NEW_BUFFER
else
__RESH_HISTNO=$((__RESH_HISTNO-1))
__RESH_HISTNO_MAX=$__RESH_HISTNO
fi
fi
# run post helper
__resh_helper_arrow_post

Write Preview
Loading…
Cancel
Save