diff --git a/.goreleaser.yml b/.goreleaser.yml index 11c048d..4cc663d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -76,55 +76,37 @@ builds: - arm - arm64 - - id: "evaluate" - main: ./cmd/evaluate - binary: bin/resh-evaluate - goarch: - - 386 - - amd64 - - arm - - arm64 - - - id: "event" - main: ./cmd/event - binary: bin/resh-event - goarch: - - 386 - - amd64 - - arm - - arm64 - - - id: "inspect" - main: ./cmd/inspect - binary: bin/resh-inspect + id: "postcollect" + main: ./cmd/postcollect + binary: bin/resh-postcollect goarch: - 386 - amd64 - arm - arm64 - - id: "postcollect" - main: ./cmd/postcollect - binary: bin/resh-postcollect + id: "session-init" + main: ./cmd/session-init + binary: bin/resh-session-init goarch: - 386 - amd64 - arm - arm64 - - id: "sanitize" - main: ./cmd/sanitize - binary: bin/resh-sanitize - goarch: + id: "config-setup" + main: ./cmd/config-setup + binary: bin/resh-config-setup + goarch: - 386 - amd64 - arm - arm64 - - id: "session-init" - main: ./cmd/session-init - binary: bin/resh-session-init - goarch: + id: "install-utils" + main: ./cmd/install-utils + binary: bin/resh-install-utils + goarch: - 386 - amd64 - arm diff --git a/Makefile b/Makefile index 02bb7e0..e5b5c87 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,13 @@ SHELL=/bin/bash LATEST_TAG=$(shell git describe --tags) -REVISION=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_revision") +COMMIT=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_commit") VERSION="${LATEST_TAG}-DEV" -GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${REVISION}" +GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.development=true" -build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect bin/resh-daemon\ - bin/resh-evaluate bin/resh-sanitize bin/resh-control bin/resh-config bin/resh-inspect bin/resh-cli +build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect\ + bin/resh-daemon bin/resh-control bin/resh-config bin/resh-cli\ + bin/resh-install-utils install: build scripts/install.sh @@ -21,13 +22,14 @@ rebuild: make build clean: - rm -f bin/resh-* + rm -f bin/* uninstall: # Uninstalling ... -rm -rf ~/.resh/ -bin/resh-%: cmd/%/*.go pkg/*/*.go cmd/control/cmd/*.go cmd/control/status/status.go +go_files = $(shell find -name '*.go') +bin/resh-%: $(go_files) grep $@ .goreleaser.yml -q # all build targets need to be included in .goreleaser.yml go build ${GOFLAGS} -o $@ cmd/$*/*.go diff --git a/cmd/cli/main.go b/cmd/cli/main.go index a58ff10..33b450f 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -6,8 +6,7 @@ import ( "errors" "flag" "fmt" - "io/ioutil" - "log" + "io" "net/http" "os" "sort" @@ -15,59 +14,41 @@ import ( "sync" "time" - "github.com/BurntSushi/toml" "github.com/awesome-gocui/gocui" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/msg" - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/searchapp" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/msg" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/internal/searchapp" + "go.uber.org/zap" - "os/user" - "path/filepath" "strconv" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string +var developement bool // special constant recognized by RESH wrappers const exitCodeExecute = 111 -var debug bool - func main() { - output, exitCode := runReshCli() + config, errCfg := cfg.New() + logger, _ := logger.New("search-app", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) + } + out := output.New(logger, "resh-search-app ERROR") + + output, exitCode := runReshCli(out, config) fmt.Print(output) os.Exit(exitCode) } -func runReshCli() (string, int) { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - logPath := filepath.Join(dir, ".resh/cli.log") - - f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - log.Fatal("Error opening file:", err) - } - defer f.Close() - - log.SetOutput(f) - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) - } - if config.Debug { - debug = true - log.SetFlags(log.LstdFlags | log.Lmicroseconds) - log.Println("DEBUG is ON") - } - +func runReshCli(out *output.Output, config cfg.Config) (string, int) { sessionID := flag.String("sessionID", "", "resh generated session id") host := flag.String("host", "", "host") pwd := flag.String("pwd", "", "present working directory") @@ -77,22 +58,23 @@ func runReshCli() (string, int) { testHistoryLines := flag.Int("test-lines", 0, "the number of lines to load from a file passed with --test-history (for testing purposes only!)") flag.Parse() + errMsg := "Failed to get necessary command-line arguments" if *sessionID == "" { - log.Println("Error: you need to specify sessionId") + out.Fatal(errMsg, errors.New("missing option --sessionId")) } if *host == "" { - log.Println("Error: you need to specify HOST") + out.Fatal(errMsg, errors.New("missing option --host")) } if *pwd == "" { - log.Println("Error: you need to specify PWD") + out.Fatal(errMsg, errors.New("missing option --pwd")) } if *gitOriginRemote == "DEFAULT" { - log.Println("Error: you need to specify gitOriginRemote") + out.Fatal(errMsg, errors.New("missing option --gitOriginRemote")) } g, err := gocui.NewGui(gocui.OutputNormal, false) if err != nil { - log.Panicln(err) + out.Fatal("Failed to launch TUI", err) } defer g.Close() @@ -107,75 +89,77 @@ func runReshCli() (string, int) { SessionID: *sessionID, PWD: *pwd, } - resp = SendCliMsg(mess, strconv.Itoa(config.Port)) + resp = SendCliMsg(out, mess, strconv.Itoa(config.Port)) } else { - resp = searchapp.LoadHistoryFromFile(*testHistory, *testHistoryLines) + resp = searchapp.LoadHistoryFromFile(out.Logger.Sugar(), *testHistory, *testHistoryLines) } st := state{ // lock sync.Mutex - cliRecords: resp.CliRecords, + cliRecords: resp.Records, initialQuery: *query, } layout := manager{ + out: out, + config: config, sessionID: *sessionID, host: *host, pwd: *pwd, - gitOriginRemote: records.NormalizeGitRemote(*gitOriginRemote), - config: config, + gitOriginRemote: *gitOriginRemote, s: &st, } g.SetManager(layout) + errMsg = "Failed to set keybindings" if err := g.SetKeybinding("", gocui.KeyTab, gocui.ModNone, layout.Next); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyArrowDown, gocui.ModNone, layout.Next); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlN, gocui.ModNone, layout.Next); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyArrowUp, gocui.ModNone, layout.Prev); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlP, gocui.ModNone, layout.Prev); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyArrowRight, gocui.ModNone, layout.SelectPaste); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyEnter, gocui.ModNone, layout.SelectExecute); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlG, gocui.ModNone, layout.AbortPaste); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlD, gocui.ModNone, quit); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlR, gocui.ModNone, layout.SwitchModes); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } layout.UpdateData(*query) layout.UpdateRawData(*query) err = g.MainLoop() if err != nil && !errors.Is(err, gocui.ErrQuit) { - log.Panicln(err) + out.Fatal("Main application loop finished with error", err) } return layout.s.output, layout.s.exitCode } type state struct { lock sync.Mutex - cliRecords []records.CliRecord + cliRecords []recordint.SearchApp data []searchapp.Item rawData []searchapp.RawItem highlightedItem int @@ -190,11 +174,13 @@ type state struct { } type manager struct { + out *output.Output + config cfg.Config + sessionID string host string pwd string gitOriginRemote string - config cfg.Config s *state } @@ -254,11 +240,11 @@ type dedupRecord struct { } func (m manager) UpdateData(input string) { - if debug { - log.Println("EDIT start") - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - } + sugar := m.out.Logger.Sugar() + sugar.Debugw("Starting data update ...", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) query := searchapp.NewQueryFromString(input, m.host, m.pwd, m.gitOriginRemote, m.config.Debug) var data []searchapp.Item itemSet := make(map[string]int) @@ -268,7 +254,7 @@ func (m manager) UpdateData(input string) { itm, err := searchapp.NewItemFromRecordForQuery(rec, query, m.config.Debug) if err != nil { // records didn't match the query - // log.Println(" * continue (no match)", rec.Pwd) + // sugar.Println(" * continue (no match)", rec.Pwd) continue } if idx, ok := itemSet[itm.Key]; ok { @@ -285,9 +271,9 @@ func (m manager) UpdateData(input string) { itemSet[itm.Key] = len(data) data = append(data, itm) } - if debug { - log.Println("len(tmpdata) =", len(data)) - } + sugar.Debugw("Got new items from records for query, sorting items ...", + "itemCount", len(data), + ) sort.SliceStable(data, func(p, q int) bool { return data[p].Score > data[q].Score }) @@ -299,19 +285,18 @@ func (m manager) UpdateData(input string) { m.s.data = append(m.s.data, itm) } m.s.highlightedItem = 0 - if debug { - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - log.Println("EDIT end") - } + sugar.Debugw("Done with data update", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) } func (m manager) UpdateRawData(input string) { - if m.config.Debug { - log.Println("EDIT start") - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - } + sugar := m.out.Logger.Sugar() + sugar.Debugw("Starting RAW data update ...", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) query := searchapp.GetRawTermsFromString(input, m.config.Debug) var data []searchapp.RawItem itemSet := make(map[string]bool) @@ -321,20 +306,20 @@ func (m manager) UpdateRawData(input string) { itm, err := searchapp.NewRawItemFromRecordForQuery(rec, query, m.config.Debug) if err != nil { // records didn't match the query - // log.Println(" * continue (no match)", rec.Pwd) + // sugar.Println(" * continue (no match)", rec.Pwd) continue } if itemSet[itm.Key] { - // log.Println(" * continue (already present)", itm.key(), itm.pwd) + // sugar.Println(" * continue (already present)", itm.key(), itm.pwd) continue } itemSet[itm.Key] = true data = append(data, itm) - // log.Println("DATA =", itm.display) - } - if debug { - log.Println("len(tmpdata) =", len(data)) + // sugar.Println("DATA =", itm.display) } + sugar.Debugw("Got new RAW items from records for query, sorting items ...", + "itemCount", len(data), + ) sort.SliceStable(data, func(p, q int) bool { return data[p].Score > data[q].Score }) @@ -346,11 +331,10 @@ func (m manager) UpdateRawData(input string) { m.s.rawData = append(m.s.rawData, itm) } m.s.highlightedItem = 0 - if debug { - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - log.Println("EDIT end") - } + sugar.Debugw("Done with RAW data update", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) } func (m manager) Edit(v *gocui.View, key gocui.Key, ch rune, mod gocui.Modifier) { gocui.DefaultEditor.Edit(v, key, ch, mod) @@ -398,7 +382,7 @@ func (m manager) Layout(g *gocui.Gui) error { v, err := g.SetView("input", 0, 0, maxX-1, 2, b) if err != nil && !errors.Is(err, gocui.ErrUnknownView) { - log.Panicln(err.Error()) + m.out.Fatal("Failed to set view 'input'", err) } v.Editable = true @@ -421,7 +405,7 @@ func (m manager) Layout(g *gocui.Gui) error { v, err = g.SetView("body", 0, 2, maxX-1, maxY, b) if err != nil && !errors.Is(err, gocui.ErrUnknownView) { - log.Panicln(err.Error()) + m.out.Fatal("Failed to set view 'body'", err) } v.Frame = false v.Autoscroll = false @@ -441,6 +425,7 @@ func quit(g *gocui.Gui, v *gocui.View) error { const smallTerminalTresholdWidth = 110 func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { + sugar := m.out.Logger.Sugar() maxX, maxY := g.Size() compactRenderingMode := false @@ -459,7 +444,7 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { if i == maxY { break } - ic := itm.DrawItemColumns(compactRenderingMode, debug) + ic := itm.DrawItemColumns(compactRenderingMode, m.config.Debug) data = append(data, ic) if i > maxPossibleMainViewHeight { // do not stretch columns because of results that will end up outside of the page @@ -497,7 +482,7 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { helpLineHeight := 1 const helpLine = "HELP: type to search, UP/DOWN or CTRL+P/N to select, RIGHT to edit, ENTER to execute, CTRL+G to abort, CTRL+C/D to quit; " + "FLAGS: G = this git repo, E# = exit status #" - // "TIP: when resh-cli is launched command line is used as initial search query" + // "TIP: when resh-cli is launched command line is used as initial search query" mainViewHeight := maxY - topBoxHeight - statusLineHeight - helpLineHeight m.s.displayedItemsCount = mainViewHeight @@ -505,7 +490,7 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { // header // header := getHeader() // error is expected for header - dispStr, _, _ := header.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, true, true, debug) + dispStr, _, _ := header.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, true, true, m.config.Debug) dispStr = searchapp.DoHighlightHeader(dispStr, maxX*2) v.WriteString(dispStr + "\n") @@ -513,33 +498,24 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { for index < len(data) { itm := data[index] if index >= mainViewHeight { - if debug { - log.Printf("Finished drawing page. mainViewHeight: %v, predictedMax: %v\n", - mainViewHeight, maxPossibleMainViewHeight) - } + sugar.Debugw("Reached bottom of the page while producing lines", + "mainViewHeight", mainViewHeight, + "predictedMaxViewHeight", maxPossibleMainViewHeight, + ) // page is full break } - displayStr, _, err := itm.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, false, true, debug) + displayStr, _, err := itm.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, false, true, m.config.Debug) if err != nil { - log.Printf("produceLine error: %v\n", err) + sugar.Error("Error while drawing item", zap.Error(err)) } if m.s.highlightedItem == index { // maxX * 2 because there are escape sequences that make it hard to tell the real string length displayStr = searchapp.DoHighlightString(displayStr, maxX*3) - if debug { - log.Println("### HightlightedItem string :", displayStr) - } - } else if debug { - log.Println(displayStr) } if strings.Contains(displayStr, "\n") { - log.Println("display string contained \\n") displayStr = strings.ReplaceAll(displayStr, "\n", "#") - if debug { - log.Println("display string contained \\n") - } } v.WriteString(displayStr + "\n") index++ @@ -553,59 +529,46 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { v.WriteString(line) } v.WriteString(helpLine) - if debug { - log.Println("len(data) =", len(m.s.data)) - log.Println("highlightedItem =", m.s.highlightedItem) - } + sugar.Debugw("Done drawing page", + "itemCount", len(m.s.data), + "highlightedItemIndex", m.s.highlightedItem, + ) return nil } func (m manager) rawMode(g *gocui.Gui, v *gocui.View) error { + sugar := m.out.Logger.Sugar() maxX, maxY := g.Size() topBoxSize := 3 m.s.displayedItemsCount = maxY - topBoxSize for i, itm := range m.s.rawData { if i == maxY { - if debug { - log.Println(maxY) - } break } displayStr := itm.CmdLineWithColor if m.s.highlightedItem == i { // use actual min requried length instead of 420 constant displayStr = searchapp.DoHighlightString(displayStr, maxX*2) - if debug { - log.Println("### HightlightedItem string :", displayStr) - } - } else if debug { - log.Println(displayStr) } if strings.Contains(displayStr, "\n") { - log.Println("display string contained \\n") displayStr = strings.ReplaceAll(displayStr, "\n", "#") - if debug { - log.Println("display string contained \\n") - } } v.WriteString(displayStr + "\n") - // if m.s.highlightedItem == i { - // v.SetHighlight(m.s.highlightedItem, true) - // } - } - if debug { - log.Println("len(data) =", len(m.s.data)) - log.Println("highlightedItem =", m.s.highlightedItem) } + sugar.Debugw("Done drawing page in RAW mode", + "itemCount", len(m.s.data), + "highlightedItemIndex", m.s.highlightedItem, + ) return nil } // SendCliMsg to daemon -func SendCliMsg(m msg.CliMsg, port string) msg.CliResponse { +func SendCliMsg(out *output.Output, m msg.CliMsg, port string) msg.CliResponse { + sugar := out.Logger.Sugar() recJSON, err := json.Marshal(m) if err != nil { - log.Fatalf("Failed to marshal message: %v\n", err) + out.Fatal("Failed to marshal message", err) } req, err := http.NewRequest( @@ -613,7 +576,7 @@ func SendCliMsg(m msg.CliMsg, port string) msg.CliResponse { "http://localhost:"+port+"/dump", bytes.NewBuffer(recJSON)) if err != nil { - log.Fatalf("Failed to build request: %v\n", err) + out.Fatal("Failed to build request", err) } req.Header.Set("Content-Type", "application/json") @@ -622,22 +585,22 @@ func SendCliMsg(m msg.CliMsg, port string) msg.CliResponse { } resp, err := client.Do(req) if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") + out.FatalDaemonNotRunning(err) } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalf("Read response error: %v\n", err) + out.Fatal("Failed read response", err) } - // log.Println(string(body)) + // sugar.Println(string(body)) response := msg.CliResponse{} err = json.Unmarshal(body, &response) if err != nil { - log.Fatalf("Unmarshal resp error: %v\n", err) - } - if debug { - log.Printf("Recieved %d records from daemon\n", len(response.CliRecords)) + out.Fatal("Failed decode response", err) } + sugar.Debugw("Recieved records from daemon", + "recordCount", len(response.Records), + ) return response } diff --git a/cmd/collect/main.go b/cmd/collect/main.go index d2d2815..230fda8 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -3,42 +3,35 @@ package main import ( "flag" "fmt" - "log" "os" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/collect" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" + "go.uber.org/zap" // "os/exec" - "os/user" + "path/filepath" "strconv" ) -// version tag from git set during build +// info passed during build var version string - -// Commit hash from git set during build var commit string +var developement bool func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") - - machineIDPath := "/etc/machine-id" - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) + config, errCfg := cfg.New() + logger, _ := logger.New("collect", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) } - // recall command - recall := flag.Bool("recall", false, "Recall command on position --histno") - recallHistno := flag.Int("histno", 0, "Recall command on position --histno") - recallPrefix := flag.String("prefix-search", "", "Recall command based on prefix --prefix-search") + out := output.New(logger, "resh-collect ERROR") // version showVersion := flag.Bool("version", false, "Show version and exit") @@ -49,60 +42,27 @@ func main() { // core cmdLine := flag.String("cmdLine", "", "command line") - exitCode := flag.Int("exitCode", -1, "exit code") - shell := flag.String("shell", "", "actual shell") - uname := flag.String("uname", "", "uname") - sessionID := flag.String("sessionId", "", "resh generated session id") - recordID := flag.String("recordId", "", "resh generated record id") - - // recall metadata - recallActions := flag.String("recall-actions", "", "recall actions that took place before executing the command") - recallStrategy := flag.String("recall-strategy", "", "recall strategy used during recall actions") - recallLastCmdLine := flag.String("recall-last-cmdline", "", "last recalled cmdline") - - // posix variables - cols := flag.String("cols", "-1", "$COLUMNS") - lines := flag.String("lines", "-1", "$LINES") + home := flag.String("home", "", "$HOME") - lang := flag.String("lang", "", "$LANG") - lcAll := flag.String("lcAll", "", "$LC_ALL") - login := flag.String("login", "", "$LOGIN") - // path := flag.String("path", "", "$PATH") pwd := flag.String("pwd", "", "$PWD - present working directory") - shellEnv := flag.String("shellEnv", "", "$SHELL") - term := flag.String("term", "", "$TERM") + + // FIXME: get device ID + deviceID := flag.String("deviceID", "", "RESH device ID") + sessionID := flag.String("sessionID", "", "resh generated session ID") + recordID := flag.String("recordID", "", "resh generated record ID") + sessionPID := flag.Int("sessionPID", -1, "PID at the start of the terminal session") + + shell := flag.String("shell", "", "current shell") + + // logname := flag.String("logname", "", "$LOGNAME") + device := flag.String("device", "", "device name, usually $HOSTNAME") // non-posix - pid := flag.Int("pid", -1, "$$") - sessionPid := flag.Int("sessionPid", -1, "$$ at session start") shlvl := flag.Int("shlvl", -1, "$SHLVL") - host := flag.String("host", "", "$HOSTNAME") - hosttype := flag.String("hosttype", "", "$HOSTTYPE") - ostype := flag.String("ostype", "", "$OSTYPE") - machtype := flag.String("machtype", "", "$MACHTYPE") - gitCdup := flag.String("gitCdup", "", "git rev-parse --show-cdup") gitRemote := flag.String("gitRemote", "", "git remote get-url origin") - gitCdupExitCode := flag.Int("gitCdupExitCode", -1, "... $?") - gitRemoteExitCode := flag.Int("gitRemoteExitCode", -1, "... $?") - - // before after - timezoneBefore := flag.String("timezoneBefore", "", "") - - osReleaseID := flag.String("osReleaseId", "", "/etc/os-release ID") - osReleaseVersionID := flag.String("osReleaseVersionId", "", - "/etc/os-release ID") - osReleaseIDLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") - osReleaseName := flag.String("osReleaseName", "", "/etc/os-release ID") - osReleasePrettyName := flag.String("osReleasePrettyName", "", - "/etc/os-release ID") - - rtb := flag.String("realtimeBefore", "-1", "before $EPOCHREALTIME") - rtsess := flag.String("realtimeSession", "-1", - "on session start $EPOCHREALTIME") - rtsessboot := flag.String("realtimeSessSinceBoot", "-1", - "on session start $EPOCHREALTIME") + time_ := flag.String("time", "-1", "$EPOCHREALTIME") flag.Parse() if *showVersion == true { @@ -114,142 +74,53 @@ func main() { os.Exit(0) } if *requireVersion != "" && *requireVersion != version { - fmt.Println("Please restart/reload this terminal session " + - "(resh version: " + version + - "; resh version of this terminal session: " + *requireVersion + - ")") - os.Exit(3) + out.FatalVersionMismatch(version, *requireVersion) } if *requireRevision != "" && *requireRevision != commit { - fmt.Println("Please restart/reload this terminal session " + - "(resh revision: " + commit + - "; resh revision of this terminal session: " + *requireRevision + - ")") - os.Exit(3) - } - if *recallPrefix != "" && *recall == false { - log.Println("Option '--prefix-search' only works with '--recall' option - exiting!") - os.Exit(4) + // this is only relevant for dev versions so we can reuse FatalVersionMismatch() + out.FatalVersionMismatch("revision "+commit, "revision "+*requireVersion) } - realtimeBefore, err := strconv.ParseFloat(*rtb, 64) - if err != nil { - log.Fatal("Flag Parsing error (rtb):", err) - } - realtimeSessionStart, err := strconv.ParseFloat(*rtsess, 64) + time, err := strconv.ParseFloat(*time_, 64) if err != nil { - log.Fatal("Flag Parsing error (rt sess):", err) + out.Fatal("Error while parsing flag --time", err) } - realtimeSessSinceBoot, err := strconv.ParseFloat(*rtsessboot, 64) - if err != nil { - log.Fatal("Flag Parsing error (rt sess boot):", err) - } - realtimeSinceSessionStart := realtimeBefore - realtimeSessionStart - realtimeSinceBoot := realtimeSessSinceBoot + realtimeSinceSessionStart - - timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(*timezoneBefore) - realtimeBeforeLocal := realtimeBefore + timezoneBeforeOffset realPwd, err := filepath.EvalSymlinks(*pwd) if err != nil { - log.Println("err while handling pwd realpath:", err) + logger.Error("Error while handling pwd realpath", zap.Error(err)) realPwd = "" } - gitDir, gitRealDir := collect.GetGitDirs(*gitCdup, *gitCdupExitCode, *pwd) - if *gitRemoteExitCode != 0 { - *gitRemote = "" - } + rec := recordint.Collect{ + SessionID: *sessionID, + Shlvl: *shlvl, + SessionPID: *sessionPID, + + Shell: *shell, + + Rec: record.V1{ + DeviceID: *deviceID, + SessionID: *sessionID, + RecordID: *recordID, + + CmdLine: *cmdLine, - // if *osReleaseID == "" { - // *osReleaseID = "linux" - // } - // if *osReleaseName == "" { - // *osReleaseName = "Linux" - // } - // if *osReleasePrettyName == "" { - // *osReleasePrettyName = "Linux" - // } - - if *recall { - rec := records.SlimRecord{ - SessionID: *sessionID, - RecallHistno: *recallHistno, - RecallPrefix: *recallPrefix, - } - str, found := collect.SendRecallRequest(rec, strconv.Itoa(config.Port)) - if found == false { - os.Exit(1) - } - fmt.Println(str) - } else { - rec := records.Record{ // posix - Cols: *cols, - Lines: *lines, - // core - BaseRecord: records.BaseRecord{ - RecallHistno: *recallHistno, - - CmdLine: *cmdLine, - ExitCode: *exitCode, - Shell: *shell, - Uname: *uname, - SessionID: *sessionID, - RecordID: *recordID, - - // posix - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - Pwd: *pwd, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - RealPwd: realPwd, - Pid: *pid, - SessionPID: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - - RealtimeBefore: realtimeBefore, - RealtimeBeforeLocal: realtimeBeforeLocal, - - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - GitDir: gitDir, - GitRealDir: gitRealDir, - GitOriginRemote: *gitRemote, - MachineID: collect.ReadFileContent(machineIDPath), - - OsReleaseID: *osReleaseID, - OsReleaseVersionID: *osReleaseVersionID, - OsReleaseIDLike: *osReleaseIDLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, - - PartOne: true, - - ReshUUID: collect.ReadFileContent(reshUUIDPath), - ReshVersion: version, - ReshRevision: commit, - - RecallActionsRaw: *recallActions, - RecallPrefix: *recallPrefix, - RecallStrategy: *recallStrategy, - RecallLastCmdLine: *recallLastCmdLine, - }, - } - collect.SendRecord(rec, strconv.Itoa(config.Port), "/record") + Home: *home, + Pwd: *pwd, + RealPwd: realPwd, + + // Logname: *logname, + Device: *device, + + GitOriginRemote: *gitRemote, + + Time: fmt.Sprintf("%.4f", time), + + PartOne: true, + PartsNotMerged: true, + }, } + collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/record") } diff --git a/cmd/config/main.go b/cmd/config/main.go index e67ee7b..a97536b 100644 --- a/cmd/config/main.go +++ b/cmd/config/main.go @@ -4,24 +4,24 @@ import ( "flag" "fmt" "os" - "os/user" - "path/filepath" "strings" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/logger" + "go.uber.org/zap" ) -func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, ".config/resh.toml") +// info passed during build +var version string +var commit string +var developement bool - var config cfg.Config - _, err := toml.DecodeFile(configPath, &config) - if err != nil { - fmt.Println("Error reading config", err) - os.Exit(1) +func main() { + config, errCfg := cfg.New() + logger, _ := logger.New("config", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) } configKey := flag.String("key", "", "Key of the requested config entry") @@ -35,24 +35,15 @@ func main() { *configKey = strings.ToLower(*configKey) switch *configKey { case "bindcontrolr": - printBoolNormalized(config.BindControlR) + fmt.Println(config.BindControlR) case "port": fmt.Println(config.Port) case "sesswatchperiodseconds": - fmt.Println(config.SesswatchPeriodSeconds) + fmt.Println(config.SessionWatchPeriodSeconds) case "sesshistinithistorysize": - fmt.Println(config.SesshistInitHistorySize) + fmt.Println(config.ReshHistoryMinSize) default: fmt.Println("Error: illegal --key!") os.Exit(1) } } - -// this might be unnecessary but I'm too lazy to look it up -func printBoolNormalized(x bool) { - if x { - fmt.Println("true") - } else { - fmt.Println("false") - } -} diff --git a/cmd/control/cmd/completion.go b/cmd/control/cmd/completion.go index e80fa56..6c0e450 100644 --- a/cmd/control/cmd/completion.go +++ b/cmd/control/cmd/completion.go @@ -3,7 +3,6 @@ package cmd import ( "os" - "github.com/curusarn/resh/cmd/control/status" "github.com/spf13/cobra" ) @@ -30,7 +29,6 @@ var completionBashCmd = &cobra.Command{ `, Run: func(cmd *cobra.Command, args []string) { rootCmd.GenBashCompletion(os.Stdout) - exitCode = status.Success }, } @@ -43,6 +41,5 @@ var completionZshCmd = &cobra.Command{ `, Run: func(cmd *cobra.Command, args []string) { rootCmd.GenZshCompletion(os.Stdout) - exitCode = status.Success }, } diff --git a/cmd/control/cmd/debug.go b/cmd/control/cmd/debug.go deleted file mode 100644 index f1401c7..0000000 --- a/cmd/control/cmd/debug.go +++ /dev/null @@ -1,66 +0,0 @@ -package cmd - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/curusarn/resh/cmd/control/status" - "github.com/spf13/cobra" -) - -var debugCmd = &cobra.Command{ - Use: "debug", - Short: "debug utils for resh", - Long: "Reloads resh rc files. Shows logs and output from last runs of resh", -} - -var debugReloadCmd = &cobra.Command{ - Use: "reload", - Short: "reload resh rc files", - Long: "Reload resh rc files", - Run: func(cmd *cobra.Command, args []string) { - exitCode = status.ReloadRcFiles - }, -} - -var debugInspectCmd = &cobra.Command{ - Use: "inspect", - Short: "inspect session history", - Run: func(cmd *cobra.Command, args []string) { - exitCode = status.InspectSessionHistory - }, -} - -var debugOutputCmd = &cobra.Command{ - Use: "output", - Short: "shows output from last runs of resh", - Long: "Shows output from last runs of resh", - Run: func(cmd *cobra.Command, args []string) { - files := []string{ - "daemon_last_run_out.txt", - "collect_last_run_out.txt", - "postcollect_last_run_out.txt", - "session_init_last_run_out.txt", - "cli_last_run_out.txt", - } - dir := os.Getenv("__RESH_XDG_CACHE_HOME") - for _, fpath := range files { - fpath := filepath.Join(dir, fpath) - debugReadFile(fpath) - } - exitCode = status.Success - }, -} - -func debugReadFile(path string) { - fmt.Println("============================================================") - fmt.Println(" filepath:", path) - fmt.Println("============================================================") - dat, err := ioutil.ReadFile(path) - if err != nil { - fmt.Println("ERROR while reading file:", err) - } - fmt.Println(string(dat)) -} diff --git a/cmd/control/cmd/enable.go b/cmd/control/cmd/enable.go deleted file mode 100644 index 360be5b..0000000 --- a/cmd/control/cmd/enable.go +++ /dev/null @@ -1,80 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "os/user" - "path/filepath" - - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/cmd/control/status" - "github.com/curusarn/resh/pkg/cfg" - "github.com/spf13/cobra" -) - -// Enable commands - -var enableCmd = &cobra.Command{ - Use: "enable", - Short: "enable RESH features (bindings)", -} - -var enableControlRBindingCmd = &cobra.Command{ - Use: "ctrl_r_binding", - Short: "enable RESH-CLI binding for Ctrl+R", - Run: func(cmd *cobra.Command, args []string) { - exitCode = enableDisableControlRBindingGlobally(true) - if exitCode == status.Success { - exitCode = status.EnableControlRBinding - } - }, -} - -// Disable commands - -var disableCmd = &cobra.Command{ - Use: "disable", - Short: "disable RESH features (bindings)", -} - -var disableControlRBindingCmd = &cobra.Command{ - Use: "ctrl_r_binding", - Short: "disable RESH-CLI binding for Ctrl+R", - Run: func(cmd *cobra.Command, args []string) { - exitCode = enableDisableControlRBindingGlobally(false) - if exitCode == status.Success { - exitCode = status.DisableControlRBinding - } - }, -} - -func enableDisableControlRBindingGlobally(value bool) status.Code { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, ".config/resh.toml") - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - fmt.Println("Error reading config", err) - return status.Fail - } - if config.BindControlR != value { - config.BindControlR = value - - f, err := os.Create(configPath) - if err != nil { - fmt.Println("Error: Failed to create/open file:", configPath, "; error:", err) - return status.Fail - } - defer f.Close() - if err := toml.NewEncoder(f).Encode(config); err != nil { - fmt.Println("Error: Failed to encode and write the config values to hdd. error:", err) - return status.Fail - } - } - if value { - fmt.Println("RESH SEARCH app Ctrl+R binding: ENABLED") - } else { - fmt.Println("RESH SEARCH app Ctrl+R binding: DISABLED") - } - return status.Success -} diff --git a/cmd/control/cmd/root.go b/cmd/control/cmd/root.go index 35c770d..baf12b0 100644 --- a/cmd/control/cmd/root.go +++ b/cmd/control/cmd/root.go @@ -1,23 +1,20 @@ package cmd import ( - "fmt" - "log" - "os/user" - "path/filepath" - - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/cmd/control/status" - "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" "github.com/spf13/cobra" ) -// globals -var exitCode status.Code +// info passed during build var version string var commit string -var debug = false +var developement bool + +// globals var config cfg.Config +var out *output.Output var rootCmd = &cobra.Command{ Use: "reshctl", @@ -25,47 +22,28 @@ var rootCmd = &cobra.Command{ } // Execute reshctl -func Execute(ver, com string) status.Code { +func Execute(ver, com string) { version = ver commit = com - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, ".config/resh.toml") - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Println("Error reading config", err) - return status.Fail - } - if config.Debug { - debug = true - // log.SetFlags(log.LstdFlags | log.Lmicroseconds) + config, errCfg := cfg.New() + logger, _ := logger.New("reshctl", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + out = output.New(logger, "ERROR") + if errCfg != nil { + out.Error("Error while getting configuration", errCfg) } - rootCmd.AddCommand(enableCmd) - enableCmd.AddCommand(enableControlRBindingCmd) - - rootCmd.AddCommand(disableCmd) - disableCmd.AddCommand(disableControlRBindingCmd) - rootCmd.AddCommand(completionCmd) completionCmd.AddCommand(completionBashCmd) completionCmd.AddCommand(completionZshCmd) - rootCmd.AddCommand(debugCmd) - debugCmd.AddCommand(debugReloadCmd) - debugCmd.AddCommand(debugInspectCmd) - debugCmd.AddCommand(debugOutputCmd) - - rootCmd.AddCommand(statusCmd) + rootCmd.AddCommand(versionCmd) updateCmd.Flags().BoolVar(&betaFlag, "beta", false, "Update to latest version even if it's beta.") rootCmd.AddCommand(updateCmd) - rootCmd.AddCommand(sanitizeCmd) - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - return status.Fail + out.Fatal("Command ended with error", err) } - return exitCode } diff --git a/cmd/control/cmd/sanitize.go b/cmd/control/cmd/sanitize.go deleted file mode 100644 index 08f29da..0000000 --- a/cmd/control/cmd/sanitize.go +++ /dev/null @@ -1,54 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "os/exec" - "os/user" - - "github.com/curusarn/resh/cmd/control/status" - "github.com/spf13/cobra" -) - -var sanitizeCmd = &cobra.Command{ - Use: "sanitize", - Short: "produce a sanitized version of your RESH history", - Run: func(cmd *cobra.Command, args []string) { - exitCode = status.Success - usr, _ := user.Current() - dir := usr.HomeDir - - fmt.Println() - fmt.Println(" HOW IT WORKS") - fmt.Println(" In sanitized history, all sensitive information is replaced with its SHA256 hashes.") - fmt.Println() - fmt.Println("Creating sanitized history files ...") - fmt.Println(" * ~/resh_history_sanitized.json (full lengh hashes)") - execCmd := exec.Command("resh-sanitize", "-trim-hashes", "0", "--output", dir+"/resh_history_sanitized.json") - execCmd.Stdout = os.Stdout - execCmd.Stderr = os.Stderr - err := execCmd.Run() - if err != nil { - exitCode = status.Fail - } - - fmt.Println(" * ~/resh_history_sanitized_trim12.json (12 char hashes)") - execCmd = exec.Command("resh-sanitize", "-trim-hashes", "12", "--output", dir+"/resh_history_sanitized_trim12.json") - execCmd.Stdout = os.Stdout - execCmd.Stderr = os.Stderr - err = execCmd.Run() - if err != nil { - exitCode = status.Fail - } - fmt.Println() - fmt.Println("Please direct all questions and/or issues to: https://github.com/curusarn/resh/issues") - fmt.Println() - fmt.Println("Please look at the resulting sanitized history using commands below.") - fmt.Println(" * Pretty print JSON") - fmt.Println(" cat ~/resh_history_sanitized_trim12.json | jq") - fmt.Println() - fmt.Println(" * Only show commands, don't show metadata") - fmt.Println(" cat ~/resh_history_sanitized_trim12.json | jq '.[\"cmdLine\"]'") - fmt.Println() - }, -} diff --git a/cmd/control/cmd/status.go b/cmd/control/cmd/status.go deleted file mode 100644 index f12b7fa..0000000 --- a/cmd/control/cmd/status.go +++ /dev/null @@ -1,72 +0,0 @@ -package cmd - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "strconv" - - "github.com/curusarn/resh/cmd/control/status" - "github.com/curusarn/resh/pkg/msg" - "github.com/spf13/cobra" -) - -var statusCmd = &cobra.Command{ - Use: "status", - Short: "show RESH status", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("resh " + version) - fmt.Println() - fmt.Println("Resh versions ...") - fmt.Println(" * installed: " + version + " (" + commit + ")") - versionEnv, found := os.LookupEnv("__RESH_VERSION") - if found == false { - versionEnv = "UNKNOWN!" - } - commitEnv, found := os.LookupEnv("__RESH_REVISION") - if found == false { - commitEnv = "unknown" - } - fmt.Println(" * this shell session: " + versionEnv + " (" + commitEnv + ")") - - resp, err := getDaemonStatus(config.Port) - if err != nil { - fmt.Println(" * RESH-DAEMON IS NOT RUNNING") - fmt.Println(" * Please REPORT this here: https://github.com/curusarn/resh/issues") - fmt.Println(" * Please RESTART this terminal window") - exitCode = status.Fail - return - } - fmt.Println(" * daemon: " + resp.Version + " (" + resp.Commit + ")") - - if version != resp.Version || version != versionEnv { - fmt.Println(" * THERE IS A MISMATCH BETWEEN VERSIONS!") - fmt.Println(" * Please REPORT this here: https://github.com/curusarn/resh/issues") - fmt.Println(" * Please RESTART this terminal window") - } - - exitCode = status.ReshStatus - }, -} - -func getDaemonStatus(port int) (msg.StatusResponse, error) { - mess := msg.StatusResponse{} - url := "http://localhost:" + strconv.Itoa(port) + "/status" - resp, err := http.Get(url) - if err != nil { - return mess, err - } - defer resp.Body.Close() - jsn, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal("Error while reading 'daemon /status' response:", err) - } - err = json.Unmarshal(jsn, &mess) - if err != nil { - log.Fatal("Error while decoding 'daemon /status' response:", err) - } - return mess, nil -} diff --git a/cmd/control/cmd/update.go b/cmd/control/cmd/update.go index dcdb21d..6263656 100644 --- a/cmd/control/cmd/update.go +++ b/cmd/control/cmd/update.go @@ -3,10 +3,8 @@ package cmd import ( "os" "os/exec" - "os/user" "path/filepath" - "github.com/curusarn/resh/cmd/control/status" "github.com/spf13/cobra" ) @@ -15,9 +13,11 @@ var updateCmd = &cobra.Command{ Use: "update", Short: "check for updates and update RESH", Run: func(cmd *cobra.Command, args []string) { - usr, _ := user.Current() - dir := usr.HomeDir - rawinstallPath := filepath.Join(dir, ".resh/rawinstall.sh") + homeDir, err := os.UserHomeDir() + if err != nil { + out.Fatal("Could not get user home dir", err) + } + rawinstallPath := filepath.Join(homeDir, ".resh/rawinstall.sh") execArgs := []string{rawinstallPath} if betaFlag { execArgs = append(execArgs, "--beta") @@ -25,9 +25,9 @@ var updateCmd = &cobra.Command{ execCmd := exec.Command("bash", execArgs...) execCmd.Stdout = os.Stdout execCmd.Stderr = os.Stderr - err := execCmd.Run() - if err == nil { - exitCode = status.Success + err = execCmd.Run() + if err != nil { + out.Fatal("Update ended with error", err) } }, } diff --git a/cmd/control/cmd/version.go b/cmd/control/cmd/version.go new file mode 100644 index 0000000..d9228c0 --- /dev/null +++ b/cmd/control/cmd/version.go @@ -0,0 +1,81 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + + "github.com/curusarn/resh/internal/msg" + "github.com/spf13/cobra" +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "show RESH version", + Run: func(cmd *cobra.Command, args []string) { + printVersion("Installed", version, commit) + + versionEnv := getEnvVarWithDefault("__RESH_VERSION", "") + commitEnv := getEnvVarWithDefault("__RESH_REVISION", "") + printVersion("This terminal session", versionEnv, commitEnv) + + // TODO: use output.Output.Error... for these + resp, err := getDaemonStatus(config.Port) + if err != nil { + fmt.Fprintf(os.Stderr, "\nERROR: Resh-daemon didn't respond - it's probably not running.\n\n") + fmt.Fprintf(os.Stderr, "-> Try restarting this terminal window to bring resh-daemon back up.\n") + fmt.Fprintf(os.Stderr, "-> If the problem persists you can check resh-daemon logs: ~/.resh/daemon.log\n") + fmt.Fprintf(os.Stderr, "-> You can file an issue at: https://github.com/curusarn/resh/issues\n") + return + } + printVersion("Currently running daemon", resp.Version, resp.Commit) + + if version != resp.Version { + fmt.Fprintf(os.Stderr, "\nWARN: Resh-daemon is running in different version than is installed now - it looks like something went wrong during resh update.\n\n") + fmt.Fprintf(os.Stderr, "-> Kill resh-daemon and then launch a new terminal window to fix that.\n") + fmt.Fprintf(os.Stderr, " $ pkill resh-daemon\n") + fmt.Fprintf(os.Stderr, "-> You can file an issue at: https://github.com/curusarn/resh/issues\n") + return + } + if version != versionEnv { + fmt.Fprintf(os.Stderr, "\nWARN: This terminal session was started with different resh version than is installed now - it looks like you updated resh and didn't restart this terminal.\n\n") + fmt.Fprintf(os.Stderr, "-> Restart this terminal window to fix that.\n") + return + } + + }, +} + +func printVersion(title, version, commit string) { + fmt.Printf("%s: %s (commit: %s)\n", title, version, commit) +} + +func getEnvVarWithDefault(varName, defaultValue string) string { + val, found := os.LookupEnv(varName) + if !found { + return defaultValue + } + return val +} + +func getDaemonStatus(port int) (msg.StatusResponse, error) { + mess := msg.StatusResponse{} + url := "http://localhost:" + strconv.Itoa(port) + "/status" + resp, err := http.Get(url) + if err != nil { + return mess, err + } + defer resp.Body.Close() + jsn, err := io.ReadAll(resp.Body) + if err != nil { + out.Fatal("Error while reading 'daemon /status' response", err) + } + err = json.Unmarshal(jsn, &mess) + if err != nil { + out.Fatal("Error while decoding 'daemon /status' response", err) + } + return mess, nil +} diff --git a/cmd/control/main.go b/cmd/control/main.go index ecae4e0..79d7289 100644 --- a/cmd/control/main.go +++ b/cmd/control/main.go @@ -1,8 +1,6 @@ package main import ( - "os" - "github.com/curusarn/resh/cmd/control/cmd" ) @@ -13,5 +11,5 @@ var version string var commit string func main() { - os.Exit(int(cmd.Execute(version, commit))) + cmd.Execute(version, commit) } diff --git a/cmd/control/status/status.go b/cmd/control/status/status.go deleted file mode 100644 index 0d797d7..0000000 --- a/cmd/control/status/status.go +++ /dev/null @@ -1,25 +0,0 @@ -package status - -// Code - exit code of the resh-control command -type Code int - -const ( - // Success exit code - Success Code = 0 - // Fail exit code - Fail = 1 - // EnableResh exit code - tells reshctl() wrapper to enable resh - // EnableResh = 30 - - // EnableControlRBinding exit code - tells reshctl() wrapper to enable control R binding - EnableControlRBinding = 32 - - // DisableControlRBinding exit code - tells reshctl() wrapper to disable control R binding - DisableControlRBinding = 42 - // ReloadRcFiles exit code - tells reshctl() wrapper to reload shellrc resh file - ReloadRcFiles = 50 - // InspectSessionHistory exit code - tells reshctl() wrapper to take current sessionID and send /inspect request to daemon - InspectSessionHistory = 51 - // ReshStatus exit code - tells reshctl() wrapper to show RESH status (aka systemctl status) - ReshStatus = 52 -) diff --git a/cmd/daemon/dump.go b/cmd/daemon/dump.go index 375da76..d76db70 100644 --- a/cmd/daemon/dump.go +++ b/cmd/daemon/dump.go @@ -2,53 +2,46 @@ package main import ( "encoding/json" - "io/ioutil" - "log" + "github.com/curusarn/resh/internal/histcli" + "io" "net/http" - "github.com/curusarn/resh/pkg/histfile" - "github.com/curusarn/resh/pkg/msg" + "github.com/curusarn/resh/internal/msg" + "go.uber.org/zap" ) type dumpHandler struct { - histfileBox *histfile.Histfile + sugar *zap.SugaredLogger + history *histcli.Histcli } func (h *dumpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if Debug { - log.Println("/dump START") - log.Println("/dump reading body ...") - } - jsn, err := ioutil.ReadAll(r.Body) + sugar := h.sugar.With(zap.String("endpoint", "/dump")) + sugar.Debugw("Handling request, reading body ...") + jsn, err := io.ReadAll(r.Body) if err != nil { - log.Println("Error reading the body", err) + sugar.Errorw("Error reading body", "error", err) return } + sugar.Debugw("Unmarshalling record ...") mess := msg.CliMsg{} - if Debug { - log.Println("/dump unmarshaling record ...") - } err = json.Unmarshal(jsn, &mess) if err != nil { - log.Println("Decoding error:", err) - log.Println("Payload:", jsn) + sugar.Errorw("Error during unmarshalling", + "error", err, + "payload", jsn, + ) return } - if Debug { - log.Println("/dump dumping ...") - } - fullRecords := h.histfileBox.DumpCliRecords() - if err != nil { - log.Println("Dump error:", err) - } + sugar.Debugw("Getting records to send ...") - resp := msg.CliResponse{CliRecords: fullRecords.List} + resp := msg.CliResponse{Records: h.history.Dump()} jsn, err = json.Marshal(&resp) if err != nil { - log.Println("Encoding error:", err) + sugar.Errorw("Error when marshaling", "error", err) return } w.Write(jsn) - log.Println("/dump END") + sugar.Infow("Request handled") } diff --git a/cmd/daemon/kill.go b/cmd/daemon/kill.go deleted file mode 100644 index 7c403a9..0000000 --- a/cmd/daemon/kill.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "io/ioutil" - "log" - "os/exec" - "strconv" - "strings" -) - -func killDaemon(pidfile string) error { - dat, err := ioutil.ReadFile(pidfile) - if err != nil { - log.Println("Reading pid file failed", err) - } - log.Print(string(dat)) - pid, err := strconv.Atoi(strings.TrimSuffix(string(dat), "\n")) - if err != nil { - log.Fatal("Pidfile contents are malformed", err) - } - cmd := exec.Command("kill", "-s", "sigint", strconv.Itoa(pid)) - err = cmd.Run() - if err != nil { - log.Printf("Command finished with error: %v", err) - return err - } - return nil -} diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index fc7b5bd..fec397f 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -1,87 +1,141 @@ package main import ( - //"flag" - + "fmt" "io/ioutil" - "log" "os" - "os/user" + "os/exec" "path/filepath" "strconv" + "strings" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/httpclient" + "github.com/curusarn/resh/internal/logger" + "go.uber.org/zap" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string - -// Debug switch -var Debug = false +var developement bool func main() { - log.Println("Daemon starting... \n" + - "version: " + version + - " commit: " + commit) - usr, _ := user.Current() - dir := usr.HomeDir - pidfilePath := filepath.Join(dir, ".resh/resh.pid") - configPath := filepath.Join(dir, ".config/resh.toml") - reshHistoryPath := filepath.Join(dir, ".resh_history.json") - bashHistoryPath := filepath.Join(dir, ".bash_history") - zshHistoryPath := filepath.Join(dir, ".zsh_history") - logPath := filepath.Join(dir, ".resh/daemon.log") - - f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) + config, errCfg := cfg.New() + logger, err := logger.New("daemon", config.LogLevel, developement) if err != nil { - log.Fatalf("Error opening file: %v\n", err) + fmt.Printf("Error while creating logger: %v", err) } - defer f.Close() - - log.SetOutput(f) - log.SetPrefix(strconv.Itoa(os.Getpid()) + " | ") - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Printf("Error reading config: %v\n", err) - return + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) } - if config.Debug { - Debug = true - log.SetFlags(log.LstdFlags | log.Lmicroseconds) + sugar := logger.Sugar() + d := daemon{sugar: sugar} + sugar.Infow("Deamon starting ...", + "version", version, + "commit", commit, + ) + + // TODO: rethink PID file and logs location + homeDir, err := os.UserHomeDir() + if err != nil { + sugar.Fatalw("Could not get user home dir", zap.Error(err)) } + PIDFile := filepath.Join(homeDir, ".resh/resh.pid") + reshHistoryPath := filepath.Join(homeDir, ".resh_history.json") + bashHistoryPath := filepath.Join(homeDir, ".bash_history") + zshHistoryPath := filepath.Join(homeDir, ".zsh_history") - res, err := isDaemonRunning(config.Port) + sugar = sugar.With(zap.Int("daemonPID", os.Getpid())) + + res, err := d.isDaemonRunning(config.Port) if err != nil { - log.Printf("Error while checking if the daemon is runnnig"+ - " - it's probably not running: %v\n", err) + sugar.Errorw("Error while checking daemon status - "+ + "it's probably not running", "error", err) } if res { - log.Println("Daemon is already running - exiting!") + sugar.Errorw("Daemon is already running - exiting!") return } - _, err = os.Stat(pidfilePath) + _, err = os.Stat(PIDFile) if err == nil { - log.Println("Pidfile exists") + sugar.Warn("Pidfile exists") // kill daemon - err = killDaemon(pidfilePath) + err = d.killDaemon(PIDFile) if err != nil { - log.Printf("Error while killing daemon: %v\n", err) + sugar.Errorw("Could not kill daemon", + "error", err, + ) } } - err = ioutil.WriteFile(pidfilePath, []byte(strconv.Itoa(os.Getpid())), 0644) + err = ioutil.WriteFile(PIDFile, []byte(strconv.Itoa(os.Getpid())), 0644) + if err != nil { + sugar.Fatalw("Could not create pidfile", + "error", err, + "PIDFile", PIDFile, + ) + } + server := Server{ + sugar: sugar, + config: config, + reshHistoryPath: reshHistoryPath, + bashHistoryPath: bashHistoryPath, + zshHistoryPath: zshHistoryPath, + } + server.Run() + sugar.Infow("Removing PID file ...", + "PIDFile", PIDFile, + ) + err = os.Remove(PIDFile) + if err != nil { + sugar.Errorw("Could not delete PID file", "error", err) + } + sugar.Info("Shutting down ...") +} + +type daemon struct { + sugar *zap.SugaredLogger +} + +func (d *daemon) getEnvOrPanic(envVar string) string { + val, found := os.LookupEnv(envVar) + if !found { + d.sugar.Fatalw("Required env variable is not set", + "variableName", envVar, + ) + } + return val +} + +func (d *daemon) isDaemonRunning(port int) (bool, error) { + url := "http://localhost:" + strconv.Itoa(port) + "/status" + client := httpclient.New() + resp, err := client.Get(url) + if err != nil { + return false, err + } + defer resp.Body.Close() + return true, nil +} + +func (d *daemon) killDaemon(pidfile string) error { + dat, err := ioutil.ReadFile(pidfile) + if err != nil { + d.sugar.Errorw("Reading pid file failed", + "PIDFile", pidfile, + "error", err) + } + d.sugar.Infow("Successfully read PID file", "contents", string(dat)) + pid, err := strconv.Atoi(strings.TrimSuffix(string(dat), "\n")) if err != nil { - log.Fatalf("Could not create pidfile: %v\n", err) + return fmt.Errorf("could not parse PID file contents: %w", err) } - runServer(config, reshHistoryPath, bashHistoryPath, zshHistoryPath) - log.Println("main: Removing pidfile ...") - err = os.Remove(pidfilePath) + d.sugar.Infow("Successfully parsed PID", "PID", pid) + cmd := exec.Command("kill", "-s", "sigint", strconv.Itoa(pid)) + err = cmd.Run() if err != nil { - log.Printf("Could not delete pidfile: %v\n", err) + return fmt.Errorf("kill command finished with error: %w", err) } - log.Println("main: Shutdown - bye") + return nil } diff --git a/cmd/daemon/recall.go b/cmd/daemon/recall.go deleted file mode 100644 index 930537c..0000000 --- a/cmd/daemon/recall.go +++ /dev/null @@ -1,109 +0,0 @@ -package main - -import ( - "encoding/json" - "io/ioutil" - "log" - "net/http" - - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/msg" - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/sesshist" -) - -type recallHandler struct { - sesshistDispatch *sesshist.Dispatch -} - -func (h *recallHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if Debug { - log.Println("/recall START") - log.Println("/recall reading body ...") - } - jsn, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Println("Error reading the body", err) - return - } - - rec := records.SlimRecord{} - if Debug { - log.Println("/recall unmarshaling record ...") - } - err = json.Unmarshal(jsn, &rec) - if err != nil { - log.Println("Decoding error:", err) - log.Println("Payload:", jsn) - return - } - if Debug { - log.Println("/recall recalling ...") - } - found := true - cmd, err := h.sesshistDispatch.Recall(rec.SessionID, rec.RecallHistno, rec.RecallPrefix) - if err != nil { - log.Println("/recall - sess id:", rec.SessionID, " - histno:", rec.RecallHistno, " -> ERROR") - log.Println("Recall error:", err) - found = false - cmd = "" - } - resp := collect.SingleResponse{CmdLine: cmd, Found: found} - if Debug { - log.Println("/recall marshaling response ...") - } - jsn, err = json.Marshal(&resp) - if err != nil { - log.Println("Encoding error:", err) - log.Println("Response:", resp) - return - } - if Debug { - log.Println(string(jsn)) - log.Println("/recall writing response ...") - } - w.Write(jsn) - log.Println("/recall END - sess id:", rec.SessionID, " - histno:", rec.RecallHistno, " -> ", cmd, " (found:", found, ")") -} - -type inspectHandler struct { - sesshistDispatch *sesshist.Dispatch -} - -func (h *inspectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - log.Println("/inspect START") - log.Println("/inspect reading body ...") - jsn, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Println("Error reading the body", err) - return - } - - mess := msg.InspectMsg{} - log.Println("/inspect unmarshaling record ...") - err = json.Unmarshal(jsn, &mess) - if err != nil { - log.Println("Decoding error:", err) - log.Println("Payload:", jsn) - return - } - log.Println("/inspect recalling ...") - cmds, err := h.sesshistDispatch.Inspect(mess.SessionID, int(mess.Count)) - if err != nil { - log.Println("/inspect - sess id:", mess.SessionID, " - count:", mess.Count, " -> ERROR") - log.Println("Inspect error:", err) - return - } - resp := msg.MultiResponse{CmdLines: cmds} - log.Println("/inspect marshaling response ...") - jsn, err = json.Marshal(&resp) - if err != nil { - log.Println("Encoding error:", err) - log.Println("Response:", resp) - return - } - // log.Println(string(jsn)) - log.Println("/inspect writing response ...") - w.Write(jsn) - log.Println("/inspect END - sess id:", mess.SessionID, " - count:", mess.Count) -} diff --git a/cmd/daemon/record.go b/cmd/daemon/record.go index ee403f8..f6bea7d 100644 --- a/cmd/daemon/record.go +++ b/cmd/daemon/record.go @@ -2,46 +2,59 @@ package main import ( "encoding/json" - "io/ioutil" - "log" + "io" "net/http" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" ) +func NewRecordHandler(sugar *zap.SugaredLogger, subscribers []chan recordint.Collect) recordHandler { + return recordHandler{ + sugar: sugar.With(zap.String("endpoint", "/record")), + subscribers: subscribers, + } +} + type recordHandler struct { - subscribers []chan records.Record + sugar *zap.SugaredLogger + subscribers []chan recordint.Collect } func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sugar := h.sugar.With(zap.String("endpoint", "/record")) + sugar.Debugw("Handling request, sending response, reading body ...") w.Write([]byte("OK\n")) - jsn, err := ioutil.ReadAll(r.Body) + jsn, err := io.ReadAll(r.Body) // run rest of the handler as goroutine to prevent any hangups go func() { if err != nil { - log.Println("Error reading the body", err) + sugar.Errorw("Error reading body", "error", err) return } - record := records.Record{} - err = json.Unmarshal(jsn, &record) + sugar.Debugw("Unmarshalling record ...") + rec := recordint.Collect{} + err = json.Unmarshal(jsn, &rec) if err != nil { - log.Println("Decoding error: ", err) - log.Println("Payload: ", jsn) + sugar.Errorw("Error during unmarshalling", + "error", err, + "payload", jsn, + ) return } - for _, sub := range h.subscribers { - sub <- record - } part := "2" - if record.PartOne { + if rec.Rec.PartOne { part = "1" } - log.Println("/record - ", record.CmdLine, " - part", part) + sugar := sugar.With( + "cmdLine", rec.Rec.CmdLine, + "part", part, + ) + sugar.Debugw("Got record, sending to subscribers ...") + for _, sub := range h.subscribers { + sub <- rec + } + sugar.Debugw("Record sent to subscribers") }() - - // fmt.Println("cmd:", r.CmdLine) - // fmt.Println("pwd:", r.Pwd) - // fmt.Println("git:", r.GitWorkTree) - // fmt.Println("exit_code:", r.ExitCode) } diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index fc70825..eeb080f 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -1,36 +1,44 @@ package main import ( + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/syncconnector" "net/http" "os" "strconv" + "time" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/histfile" - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/sesshist" - "github.com/curusarn/resh/pkg/sesswatch" - "github.com/curusarn/resh/pkg/signalhandler" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/histfile" + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/internal/sesswatch" + "github.com/curusarn/resh/internal/signalhandler" + "go.uber.org/zap" ) -func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPath string) { - var recordSubscribers []chan records.Record - var sessionInitSubscribers []chan records.Record +// TODO: turn server and handlers into package + +type Server struct { + sugar *zap.SugaredLogger + config cfg.Config + + reshHistoryPath string + bashHistoryPath string + zshHistoryPath string +} + +func (s *Server) Run() { + var recordSubscribers []chan recordint.Collect + var sessionInitSubscribers []chan recordint.SessionInit var sessionDropSubscribers []chan string var signalSubscribers []chan os.Signal shutdown := make(chan string) - // sessshist - sesshistSessionsToInit := make(chan records.Record) - sessionInitSubscribers = append(sessionInitSubscribers, sesshistSessionsToInit) - sesshistSessionsToDrop := make(chan string) - sessionDropSubscribers = append(sessionDropSubscribers, sesshistSessionsToDrop) - sesshistRecords := make(chan records.Record) - recordSubscribers = append(recordSubscribers, sesshistRecords) + history := histcli.New(s.sugar) // histfile - histfileRecords := make(chan records.Record) + histfileRecords := make(chan recordint.Collect) recordSubscribers = append(recordSubscribers, histfileRecords) histfileSessionsToDrop := make(chan string) sessionDropSubscribers = append(sessionDropSubscribers, histfileSessionsToDrop) @@ -38,38 +46,55 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa signalSubscribers = append(signalSubscribers, histfileSignals) maxHistSize := 10000 // lines minHistSizeKB := 2000 // roughly lines - histfileBox := histfile.New(histfileRecords, histfileSessionsToDrop, - reshHistoryPath, bashHistoryPath, zshHistoryPath, + histfile.New(s.sugar, histfileRecords, histfileSessionsToDrop, + s.reshHistoryPath, s.bashHistoryPath, s.zshHistoryPath, maxHistSize, minHistSizeKB, - histfileSignals, shutdown) - - // sesshist New - sesshistDispatch := sesshist.NewDispatch(sesshistSessionsToInit, sesshistSessionsToDrop, - sesshistRecords, histfileBox, - config.SesshistInitHistorySize) + histfileSignals, shutdown, history) // sesswatch - sesswatchRecords := make(chan records.Record) + sesswatchRecords := make(chan recordint.Collect) + // TODO: add sync connector subscriber recordSubscribers = append(recordSubscribers, sesswatchRecords) - sesswatchSessionsToWatch := make(chan records.Record) - sessionInitSubscribers = append(sessionInitSubscribers, sesswatchRecords, sesswatchSessionsToWatch) - sesswatch.Go(sesswatchSessionsToWatch, sesswatchRecords, sessionDropSubscribers, config.SesswatchPeriodSeconds) + sesswatchSessionsToWatch := make(chan recordint.SessionInit) + sessionInitSubscribers = append(sessionInitSubscribers, sesswatchSessionsToWatch) + sesswatch.Go( + s.sugar, + sesswatchSessionsToWatch, + sesswatchRecords, + sessionDropSubscribers, + s.config.SessionWatchPeriodSeconds, + ) // handlers mux := http.NewServeMux() - mux.HandleFunc("/status", statusHandler) - mux.Handle("/record", &recordHandler{subscribers: recordSubscribers}) - mux.Handle("/session_init", &sessionInitHandler{subscribers: sessionInitSubscribers}) - mux.Handle("/recall", &recallHandler{sesshistDispatch: sesshistDispatch}) - mux.Handle("/inspect", &inspectHandler{sesshistDispatch: sesshistDispatch}) - mux.Handle("/dump", &dumpHandler{histfileBox: histfileBox}) + mux.Handle("/status", &statusHandler{sugar: s.sugar}) + mux.Handle("/record", &recordHandler{sugar: s.sugar, subscribers: recordSubscribers}) + mux.Handle("/session_init", &sessionInitHandler{sugar: s.sugar, subscribers: sessionInitSubscribers}) + mux.Handle("/dump", &dumpHandler{sugar: s.sugar, history: history}) server := &http.Server{ - Addr: "localhost:" + strconv.Itoa(config.Port), - Handler: mux, + Addr: "localhost:" + strconv.Itoa(s.config.Port), + Handler: mux, + ReadTimeout: 1 * time.Second, + WriteTimeout: 1 * time.Second, + ReadHeaderTimeout: 1 * time.Second, + IdleTimeout: 30 * time.Second, } go server.ListenAndServe() + s.sugar.Infow("", "sync_addr", s.config.SyncConnectorAddress) + if s.config.SyncConnectorAddress != nil { + sc, err := syncconnector.New(s.sugar, *s.config.SyncConnectorAddress, s.config.SyncConnectorAuthToken, s.config.SyncConnectorPullPeriodSeconds, s.config.SyncConnectorSendPeriodSeconds, history) + if err != nil { + s.sugar.Errorw("Sync Connector init failed", "error", err) + } else { + s.sugar.Infow("Initialized Sync Connector", "Sync Connector", sc) + } + // TODO: load sync connector data + // TODO: load sync connector data + // TODO: send connector data periodically (record by record / or batch) + } + // signalhandler - takes over the main goroutine so when signal handler exists the whole program exits - signalhandler.Run(signalSubscribers, shutdown, server) + signalhandler.Run(s.sugar, signalSubscribers, shutdown, server) } diff --git a/cmd/daemon/session-init.go b/cmd/daemon/session-init.go index 27a1b27..5ee20c8 100644 --- a/cmd/daemon/session-init.go +++ b/cmd/daemon/session-init.go @@ -2,37 +2,49 @@ package main import ( "encoding/json" - "io/ioutil" - "log" + "io" "net/http" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" ) type sessionInitHandler struct { - subscribers []chan records.Record + sugar *zap.SugaredLogger + subscribers []chan recordint.SessionInit } func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sugar := h.sugar.With(zap.String("endpoint", "/session_init")) + sugar.Debugw("Handling request, sending response, reading body ...") w.Write([]byte("OK\n")) - jsn, err := ioutil.ReadAll(r.Body) + // TODO: should we somehow check for errors here? + jsn, err := io.ReadAll(r.Body) // run rest of the handler as goroutine to prevent any hangups go func() { if err != nil { - log.Println("Error reading the body", err) + sugar.Errorw("Error reading body", "error", err) return } - record := records.Record{} - err = json.Unmarshal(jsn, &record) + sugar.Debugw("Unmarshalling record ...") + rec := recordint.SessionInit{} + err = json.Unmarshal(jsn, &rec) if err != nil { - log.Println("Decoding error: ", err) - log.Println("Payload: ", jsn) + sugar.Errorw("Error during unmarshalling", + "error", err, + "payload", jsn, + ) return } + sugar := sugar.With( + "sessionID", rec.SessionID, + "sessionPID", rec.SessionPID, + ) + sugar.Infow("Got session, sending to subscribers ...") for _, sub := range h.subscribers { - sub <- record + sub <- rec } - log.Println("/session_init - id:", record.SessionID, " - pid:", record.SessionPID) + sugar.Debugw("Session sent to subscribers") }() } diff --git a/cmd/daemon/status.go b/cmd/daemon/status.go index 599b8dd..3c6b416 100644 --- a/cmd/daemon/status.go +++ b/cmd/daemon/status.go @@ -2,16 +2,19 @@ package main import ( "encoding/json" - "log" "net/http" - "strconv" - "github.com/curusarn/resh/pkg/httpclient" - "github.com/curusarn/resh/pkg/msg" + "github.com/curusarn/resh/internal/msg" + "go.uber.org/zap" ) -func statusHandler(w http.ResponseWriter, r *http.Request) { - log.Println("/status START") +type statusHandler struct { + sugar *zap.SugaredLogger +} + +func (h *statusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sugar := h.sugar.With(zap.String("endpoint", "/status")) + sugar.Debugw("Handling request ...") resp := msg.StatusResponse{ Status: true, Version: version, @@ -19,23 +22,12 @@ func statusHandler(w http.ResponseWriter, r *http.Request) { } jsn, err := json.Marshal(&resp) if err != nil { - log.Println("Encoding error:", err) - log.Println("Response:", resp) + sugar.Errorw("Error when marshaling", + "error", err, + "response", resp, + ) return } w.Write(jsn) - log.Println("/status END") -} - -func isDaemonRunning(port int) (bool, error) { - url := "http://localhost:" + strconv.Itoa(port) + "/status" - client := httpclient.New() - resp, err := client.Get(url) - if err != nil { - log.Printf("Error while checking daemon status - "+ - "it's probably not running: %v\n", err) - return false, err - } - defer resp.Body.Close() - return true, nil + sugar.Infow("Request handled") } diff --git a/cmd/evaluate/main.go b/cmd/evaluate/main.go deleted file mode 100644 index 5b4bda8..0000000 --- a/cmd/evaluate/main.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "os/user" - "path/filepath" - - "github.com/curusarn/resh/pkg/histanal" - "github.com/curusarn/resh/pkg/strat" -) - -// version from git set during build -var version string - -// commit from git set during build -var commit string - -func main() { - const maxCandidates = 50 - - usr, _ := user.Current() - dir := usr.HomeDir - historyPath := filepath.Join(dir, ".resh_history.json") - historyPathBatchMode := filepath.Join(dir, "resh_history.json") - sanitizedHistoryPath := filepath.Join(dir, "resh_history_sanitized.json") - // tmpPath := "/tmp/resh-evaluate-tmp.json" - - showVersion := flag.Bool("version", false, "Show version and exit") - showRevision := flag.Bool("revision", false, "Show git revision and exit") - input := flag.String("input", "", - "Input file (default: "+historyPath+" OR "+sanitizedHistoryPath+ - " depending on --sanitized-input option)") - // outputDir := flag.String("output", "/tmp/resh-evaluate", "Output directory") - sanitizedInput := flag.Bool("sanitized-input", false, - "Handle input as sanitized (also changes default value for input argument)") - plottingScript := flag.String("plotting-script", "resh-evaluate-plot.py", "Script to use for plotting") - inputDataRoot := flag.String("input-data-root", "", - "Input data root, enables batch mode, looks for files matching --input option") - slow := flag.Bool("slow", false, - "Enables strategies that takes a long time (e.g. markov chain strategies).") - skipFailedCmds := flag.Bool("skip-failed-cmds", false, - "Skips records with non-zero exit status.") - debugRecords := flag.Float64("debug", 0, "Debug records - percentage of records that should be debugged.") - - flag.Parse() - - // handle show{Version,Revision} options - if *showVersion == true { - fmt.Println(version) - os.Exit(0) - } - if *showRevision == true { - fmt.Println(commit) - os.Exit(0) - } - - // handle batch mode - batchMode := false - if *inputDataRoot != "" { - batchMode = true - } - // set default input - if *input == "" { - if *sanitizedInput { - *input = sanitizedHistoryPath - } else if batchMode { - *input = historyPathBatchMode - } else { - *input = historyPath - } - } - - var evaluator histanal.HistEval - if batchMode { - evaluator = histanal.NewHistEvalBatchMode(*input, *inputDataRoot, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) - } else { - evaluator = histanal.NewHistEval(*input, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) - } - - var simpleStrategies []strat.ISimpleStrategy - var strategies []strat.IStrategy - - // dummy := strategyDummy{} - // simpleStrategies = append(simpleStrategies, &dummy) - - simpleStrategies = append(simpleStrategies, &strat.Recent{}) - - // frequent := strategyFrequent{} - // frequent.init() - // simpleStrategies = append(simpleStrategies, &frequent) - - // random := strategyRandom{candidatesSize: maxCandidates} - // random.init() - // simpleStrategies = append(simpleStrategies, &random) - - directory := strat.DirectorySensitive{} - directory.Init() - simpleStrategies = append(simpleStrategies, &directory) - - // dynamicDistG := strat.DynamicRecordDistance{ - // MaxDepth: 3000, - // DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1, Git: 10}, - // Label: "10*pwd,10*realpwd,session,time,10*git", - // } - // dynamicDistG.Init() - // strategies = append(strategies, &dynamicDistG) - - // NOTE: this is the decent one !!! - // distanceStaticBest := strat.RecordDistance{ - // MaxDepth: 3000, - // DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, - // Label: "10*pwd,10*realpwd,session,time", - // } - // strategies = append(strategies, &distanceStaticBest) - - recentBash := strat.RecentBash{} - recentBash.Init() - strategies = append(strategies, &recentBash) - - if *slow { - - markovCmd := strat.MarkovChainCmd{Order: 1} - markovCmd.Init() - - markovCmd2 := strat.MarkovChainCmd{Order: 2} - markovCmd2.Init() - - markov := strat.MarkovChain{Order: 1} - markov.Init() - - markov2 := strat.MarkovChain{Order: 2} - markov2.Init() - - simpleStrategies = append(simpleStrategies, &markovCmd2, &markovCmd, &markov2, &markov) - } - - for _, strategy := range simpleStrategies { - strategies = append(strategies, strat.NewSimpleStrategyWrapper(strategy)) - } - - for _, strat := range strategies { - err := evaluator.Evaluate(strat) - if err != nil { - log.Println("Evaluator evaluate() error:", err) - } - } - - evaluator.CalculateStatsAndPlot(*plottingScript) -} diff --git a/cmd/event/main.go b/cmd/event/main.go deleted file mode 100644 index fe3cb72..0000000 --- a/cmd/event/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "fmt" - -func main() { - fmt.Println("Hell world") -} diff --git a/cmd/inspect/main.go b/cmd/inspect/main.go deleted file mode 100644 index 7b76a40..0000000 --- a/cmd/inspect/main.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "time" - - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/msg" - - "os/user" - "path/filepath" - "strconv" -) - -// version from git set during build -var version string - -// commit from git set during build -var commit string - -func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) - } - - sessionID := flag.String("sessionID", "", "resh generated session id") - count := flag.Uint("count", 10, "Number of cmdLines to return") - flag.Parse() - - if *sessionID == "" { - fmt.Println("Error: you need to specify sessionId") - } - - m := msg.InspectMsg{SessionID: *sessionID, Count: *count} - resp := SendInspectMsg(m, strconv.Itoa(config.Port)) - for _, cmdLine := range resp.CmdLines { - fmt.Println("`" + cmdLine + "'") - } -} - -// SendInspectMsg to daemon -func SendInspectMsg(m msg.InspectMsg, port string) msg.MultiResponse { - recJSON, err := json.Marshal(m) - if err != nil { - log.Fatal("send err 1", err) - } - - req, err := http.NewRequest("POST", "http://localhost:"+port+"/inspect", - bytes.NewBuffer(recJSON)) - if err != nil { - log.Fatal("send err 2", err) - } - req.Header.Set("Content-Type", "application/json") - - client := http.Client{ - Timeout: 3 * time.Second, - } - resp, err := client.Do(req) - if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal("read response error") - } - // log.Println(string(body)) - response := msg.MultiResponse{} - err = json.Unmarshal(body, &response) - if err != nil { - log.Fatal("unmarshal resp error: ", err) - } - return response -} diff --git a/cmd/install-utils/backup.go b/cmd/install-utils/backup.go new file mode 100644 index 0000000..1121535 --- /dev/null +++ b/cmd/install-utils/backup.go @@ -0,0 +1,14 @@ +package main + +func backup() { + panic("Backup not implemented yet!") + // Backup ~/.resh + // Backup xdg_data/resh/history.reshjson + // TODO: figure out history file localtions when using history sync +} + +func rollback() { + panic("Rollback not implemented yet!") + // Rollback ~/.resh + // Rollback history +} diff --git a/cmd/install-utils/main.go b/cmd/install-utils/main.go new file mode 100644 index 0000000..0d09a30 --- /dev/null +++ b/cmd/install-utils/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "fmt" + "os" +) + +// info passed during build +var version string +var commit string +var developement bool + +func main() { + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "ERROR: Not eonugh arguments\n") + printUsage(os.Stderr) + } + command := os.Args[1] + switch command { + case "backup": + backup() + case "rollback": + rollback() + case "migrate-config": + migrateConfig() + case "migrate-history": + migrateHistory() + case "help": + printUsage(os.Stdout) + default: + fmt.Fprintf(os.Stderr, "ERROR: Unknown command: %s\n", command) + printUsage(os.Stderr) + } +} + +func printUsage(f *os.File) { + usage := ` +USAGE: ./install-utils COMMAND +Utils used during RESH instalation. + +COMMANDS: + backup backup resh installation and data + rollback restore resh installation and data from backup + migrate-config update config to reflect updates + migrate-history update history to reflect updates + help show this help + +` + fmt.Fprintf(f, usage) +} diff --git a/cmd/install-utils/migrate.go b/cmd/install-utils/migrate.go new file mode 100644 index 0000000..e68ffe2 --- /dev/null +++ b/cmd/install-utils/migrate.go @@ -0,0 +1,40 @@ +package main + +import ( + "fmt" + "os" + + "github.com/curusarn/resh/internal/cfg" +) + +func migrateConfig() { + err := cfg.Touch() + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to touch config file: %v\n", err) + os.Exit(1) + } + changes, err := cfg.Migrate() + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to update config file: %v\n", err) + os.Exit(1) + } + if changes { + fmt.Printf("RESH config file format has changed since last update - your config was updated to reflect the changes.\n") + } +} + +func migrateHistory() { + // homeDir, err := os.UserHomeDir() + // if err != nil { + + // } + + // TODO: Find history in: + // - .resh/history.json (copy) - message user to delete the file once they confirm the new setup works + // - .resh_history.json (copy) - message user to delete the file once they confirm the new setup works + // - xdg_data/resh/history.reshjson + + // Read xdg_data/resh/history.reshjson + // Write xdg_data/resh/history.reshjson + // the old format can be found in the backup dir +} diff --git a/cmd/postcollect/main.go b/cmd/postcollect/main.go index b5cb6b0..a187808 100644 --- a/cmd/postcollect/main.go +++ b/cmd/postcollect/main.go @@ -3,69 +3,49 @@ package main import ( "flag" "fmt" - "log" "os" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/collect" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" + "go.uber.org/zap" // "os/exec" - "os/user" - "path/filepath" + "strconv" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string +var developement bool func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") - - machineIDPath := "/etc/machine-id" - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) + config, errCfg := cfg.New() + logger, _ := logger.New("postcollect", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) } + out := output.New(logger, "resh-postcollect ERROR") + showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") requireVersion := flag.String("requireVersion", "", "abort if version doesn't match") requireRevision := flag.String("requireRevision", "", "abort if revision doesn't match") - cmdLine := flag.String("cmdLine", "", "command line") exitCode := flag.Int("exitCode", -1, "exit code") - sessionID := flag.String("sessionId", "", "resh generated session id") - recordID := flag.String("recordId", "", "resh generated record id") + sessionID := flag.String("sessionID", "", "resh generated session id") + recordID := flag.String("recordID", "", "resh generated record id") shlvl := flag.Int("shlvl", -1, "$SHLVL") - shell := flag.String("shell", "", "actual shell") - - // posix variables - pwdAfter := flag.String("pwdAfter", "", "$PWD after command") - // non-posix - // sessionPid := flag.Int("sessionPid", -1, "$$ at session start") - - gitCdupAfter := flag.String("gitCdupAfter", "", "git rev-parse --show-cdup") - gitRemoteAfter := flag.String("gitRemoteAfter", "", "git remote get-url origin") - - gitCdupExitCodeAfter := flag.Int("gitCdupExitCodeAfter", -1, "... $?") - gitRemoteExitCodeAfter := flag.Int("gitRemoteExitCodeAfter", -1, "... $?") - - // before after - timezoneAfter := flag.String("timezoneAfter", "", "") - - rtb := flag.String("realtimeBefore", "-1", "before $EPOCHREALTIME") - rta := flag.String("realtimeAfter", "-1", "after $EPOCHREALTIME") + rtb := flag.String("timeBefore", "-1", "before $EPOCHREALTIME") + rta := flag.String("timeAfter", "-1", "after $EPOCHREALTIME") flag.Parse() if *showVersion == true { @@ -77,78 +57,37 @@ func main() { os.Exit(0) } if *requireVersion != "" && *requireVersion != version { - fmt.Println("Please restart/reload this terminal session " + - "(resh version: " + version + - "; resh version of this terminal session: " + *requireVersion + - ")") - os.Exit(3) + out.FatalVersionMismatch(version, *requireVersion) } if *requireRevision != "" && *requireRevision != commit { - fmt.Println("Please restart/reload this terminal session " + - "(resh revision: " + commit + - "; resh revision of this terminal session: " + *requireRevision + - ")") - os.Exit(3) - } - realtimeAfter, err := strconv.ParseFloat(*rta, 64) - if err != nil { - log.Fatal("Flag Parsing error (rta):", err) + // this is only relevant for dev versions so we can reuse FatalVersionMismatch() + out.FatalVersionMismatch("revision "+commit, "revision "+*requireVersion) } - realtimeBefore, err := strconv.ParseFloat(*rtb, 64) + + timeAfter, err := strconv.ParseFloat(*rta, 64) if err != nil { - log.Fatal("Flag Parsing error (rtb):", err) + out.Fatal("Error while parsing flag --timeAfter", err) } - realtimeDuration := realtimeAfter - realtimeBefore - - timezoneAfterOffset := collect.GetTimezoneOffsetInSeconds(*timezoneAfter) - realtimeAfterLocal := realtimeAfter + timezoneAfterOffset - - realPwdAfter, err := filepath.EvalSymlinks(*pwdAfter) + timeBefore, err := strconv.ParseFloat(*rtb, 64) if err != nil { - log.Println("err while handling pwdAfter realpath:", err) - realPwdAfter = "" + out.Fatal("Error while parsing flag --timeBefore", err) } + duration := timeAfter - timeBefore - gitDirAfter, gitRealDirAfter := collect.GetGitDirs(*gitCdupAfter, *gitCdupExitCodeAfter, *pwdAfter) - if *gitRemoteExitCodeAfter != 0 { - *gitRemoteAfter = "" - } + // FIXME: use recordint.Postollect + rec := recordint.Collect{ + SessionID: *sessionID, + Shlvl: *shlvl, - rec := records.Record{ - // core - BaseRecord: records.BaseRecord{ - CmdLine: *cmdLine, - ExitCode: *exitCode, - SessionID: *sessionID, + Rec: record.V1{ RecordID: *recordID, - Shlvl: *shlvl, - Shell: *shell, - - PwdAfter: *pwdAfter, - - // non-posix - RealPwdAfter: realPwdAfter, - - // before after - TimezoneAfter: *timezoneAfter, - - RealtimeBefore: realtimeBefore, - RealtimeAfter: realtimeAfter, - RealtimeAfterLocal: realtimeAfterLocal, - - RealtimeDuration: realtimeDuration, - - GitDirAfter: gitDirAfter, - GitRealDirAfter: gitRealDirAfter, - GitOriginRemoteAfter: *gitRemoteAfter, - MachineID: collect.ReadFileContent(machineIDPath), + SessionID: *sessionID, - PartOne: false, + ExitCode: *exitCode, + Duration: fmt.Sprintf("%.4f", duration), - ReshUUID: collect.ReadFileContent(reshUUIDPath), - ReshVersion: version, - ReshRevision: commit, + PartsNotMerged: true, }, } - collect.SendRecord(rec, strconv.Itoa(config.Port), "/record") + collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/record") } diff --git a/cmd/sanitize/main.go b/cmd/sanitize/main.go deleted file mode 100644 index c4fd60a..0000000 --- a/cmd/sanitize/main.go +++ /dev/null @@ -1,523 +0,0 @@ -package main - -import ( - "bufio" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "encoding/json" - "errors" - "flag" - "fmt" - "log" - "math" - "net/url" - "os" - "os/user" - "path" - "path/filepath" - "strconv" - "strings" - "unicode" - - "github.com/coreos/go-semver/semver" - "github.com/curusarn/resh/pkg/records" - giturls "github.com/whilp/git-urls" -) - -// version from git set during build -var version string - -// commit from git set during build -var commit string - -func main() { - usr, _ := user.Current() - dir := usr.HomeDir - historyPath := filepath.Join(dir, ".resh_history.json") - // outputPath := filepath.Join(dir, "resh_history_sanitized.json") - sanitizerDataPath := filepath.Join(dir, ".resh", "sanitizer_data") - - showVersion := flag.Bool("version", false, "Show version and exit") - showRevision := flag.Bool("revision", false, "Show git revision and exit") - trimHashes := flag.Int("trim-hashes", 12, "Trim hashes to N characters, '0' turns off trimming") - inputPath := flag.String("input", historyPath, "Input file") - outputPath := flag.String("output", "", "Output file (default: use stdout)") - - flag.Parse() - - if *showVersion == true { - fmt.Println(version) - os.Exit(0) - } - if *showRevision == true { - fmt.Println(commit) - os.Exit(0) - } - sanitizer := sanitizer{hashLength: *trimHashes} - err := sanitizer.init(sanitizerDataPath) - if err != nil { - log.Fatal("Sanitizer init() error:", err) - } - - inputFile, err := os.Open(*inputPath) - if err != nil { - log.Fatal("Open() resh history file error:", err) - } - defer inputFile.Close() - - var writer *bufio.Writer - if *outputPath == "" { - writer = bufio.NewWriter(os.Stdout) - } else { - outputFile, err := os.Create(*outputPath) - if err != nil { - log.Fatal("Create() output file error:", err) - } - defer outputFile.Close() - writer = bufio.NewWriter(outputFile) - } - defer writer.Flush() - - scanner := bufio.NewScanner(inputFile) - for scanner.Scan() { - record := records.Record{} - fallbackRecord := records.FallbackRecord{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) - } - record = records.Convert(&fallbackRecord) - } - err = sanitizer.sanitizeRecord(&record) - if err != nil { - log.Println("Line:", line) - log.Fatal("Sanitization error:", err) - } - outLine, err := json.Marshal(&record) - if err != nil { - log.Println("Line:", line) - log.Fatal("Encoding error:", err) - } - // fmt.Println(string(outLine)) - n, err := writer.WriteString(string(outLine) + "\n") - if err != nil { - log.Fatal(err) - } - if n == 0 { - log.Fatal("Nothing was written", n) - } - } -} - -type sanitizer struct { - hashLength int - whitelist map[string]bool -} - -func (s *sanitizer) init(dataPath string) error { - globalData := path.Join(dataPath, "whitelist.txt") - s.whitelist = loadData(globalData) - return nil -} - -func loadData(fname string) map[string]bool { - file, err := os.Open(fname) - if err != nil { - log.Fatal("Open() file error:", err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - data := make(map[string]bool) - for scanner.Scan() { - line := scanner.Text() - data[line] = true - } - return data -} - -func (s *sanitizer) sanitizeRecord(record *records.Record) error { - // hash directories of the paths - record.Pwd = s.sanitizePath(record.Pwd) - record.RealPwd = s.sanitizePath(record.RealPwd) - record.PwdAfter = s.sanitizePath(record.PwdAfter) - record.RealPwdAfter = s.sanitizePath(record.RealPwdAfter) - record.GitDir = s.sanitizePath(record.GitDir) - record.GitDirAfter = s.sanitizePath(record.GitDirAfter) - record.GitRealDir = s.sanitizePath(record.GitRealDir) - record.GitRealDirAfter = s.sanitizePath(record.GitRealDirAfter) - record.Home = s.sanitizePath(record.Home) - record.ShellEnv = s.sanitizePath(record.ShellEnv) - - // hash the most sensitive info, do not tokenize - record.Host = s.hashToken(record.Host) - record.Login = s.hashToken(record.Login) - record.MachineID = s.hashToken(record.MachineID) - - var err error - // this changes git url a bit but I'm still happy with the result - // e.g. "git@github.com:curusarn/resh" becomes "ssh://git@github.com/3385162f14d7/5a7b2909005c" - // notice the "ssh://" prefix - record.GitOriginRemote, err = s.sanitizeGitURL(record.GitOriginRemote) - if err != nil { - log.Println("Error while snitizing GitOriginRemote url", record.GitOriginRemote, ":", err) - return err - } - record.GitOriginRemoteAfter, err = s.sanitizeGitURL(record.GitOriginRemoteAfter) - if err != nil { - log.Println("Error while snitizing GitOriginRemoteAfter url", record.GitOriginRemoteAfter, ":", err) - return err - } - - // sanitization destroys original CmdLine length -> save it - record.CmdLength = len(record.CmdLine) - - record.CmdLine, err = s.sanitizeCmdLine(record.CmdLine) - if err != nil { - log.Fatal("Cmd:", record.CmdLine, "; sanitization error:", err) - } - record.RecallLastCmdLine, err = s.sanitizeCmdLine(record.RecallLastCmdLine) - if err != nil { - log.Fatal("RecallLastCmdLine:", record.RecallLastCmdLine, "; sanitization error:", err) - } - - if len(record.RecallActionsRaw) > 0 { - record.RecallActionsRaw, err = s.sanitizeRecallActions(record.RecallActionsRaw, record.ReshVersion) - if err != nil { - log.Println("RecallActionsRaw:", record.RecallActionsRaw, "; sanitization error:", err) - } - } - // add a flag to signify that the record has been sanitized - record.Sanitized = true - return nil -} - -func fixSeparator(str string) string { - if len(str) > 0 && str[0] == ';' { - return "|||" + str[1:] - } - return str -} - -func minIndex(str string, substrs []string) (idx, substrIdx int) { - minMatch := math.MaxInt32 - for i, sep := range substrs { - match := strings.Index(str, sep) - if match != -1 && match < minMatch { - minMatch = match - substrIdx = i - } - } - idx = minMatch - return -} - -// sanitizes the recall actions by replacing the recall prefix with it's length -func (s *sanitizer) sanitizeRecallActions(str string, reshVersion string) (string, error) { - if len(str) == 0 { - return "", nil - } - var separators []string - seps := []string{"|||"} - refVersion, err := semver.NewVersion("2.5.14") - if err != nil { - return str, fmt.Errorf("sanitizeRecallActions: semver error: %s", err.Error()) - } - if len(reshVersion) == 0 { - return str, errors.New("sanitizeRecallActions: record.ReshVersion is an empty string") - } - if reshVersion == "dev" { - reshVersion = "0.0.0" - } - if reshVersion[0] == 'v' { - reshVersion = reshVersion[1:] - } - recordVersion, err := semver.NewVersion(reshVersion) - if err != nil { - return str, fmt.Errorf("sanitizeRecallActions: semver error: %s; version string: %s", err.Error(), reshVersion) - } - if recordVersion.LessThan(*refVersion) { - seps = append(seps, ";") - } - - actions := []string{"arrow_up", "arrow_down", "control_R"} - for _, sep := range seps { - for _, action := range actions { - separators = append(separators, sep+action+":") - } - } - /* - - find any of {|||,;}{arrow_up,arrow_down,control_R}: in the recallActions (on the lowest index) - - use found substring to parse out the next prefix - - sanitize prefix - - add fixed substring and sanitized prefix to output - */ - doBreak := false - sanStr := "" - idx := 0 - var currSeparator string - tokenLen, sepIdx := minIndex(str, separators) - if tokenLen != 0 { - return str, errors.New("sanitizeReacallActions: unexpected string before first action/separator") - } - currSeparator = separators[sepIdx] - idx += len(currSeparator) - for !doBreak { - tokenLen, sepIdx := minIndex(str[idx:], separators) - if tokenLen > len(str[idx:]) { - tokenLen = len(str[idx:]) - doBreak = true - } - // token := str[idx : idx+tokenLen] - sanStr += fixSeparator(currSeparator) + strconv.Itoa(tokenLen) - currSeparator = separators[sepIdx] - idx += tokenLen + len(currSeparator) - } - return sanStr, nil -} - -func (s *sanitizer) sanitizeCmdLine(cmdLine string) (string, error) { - const optionEndingChars = "\"$'\\#[]!><|;{}()*,?~&=`:@^/+%." // all bash control characters, '=', ... - const optionAllowedChars = "-_" // characters commonly found inside of options - sanCmdLine := "" - buff := "" - - // simple options shouldn't be sanitized - // 1) whitespace 2) "-" or "--" 3) letters, digits, "-", "_" 4) ending whitespace or any of "=;)" - var optionDetected bool - - prevR3 := ' ' - prevR2 := ' ' - prevR := ' ' - for _, r := range cmdLine { - switch optionDetected { - case true: - if unicode.IsSpace(r) || strings.ContainsRune(optionEndingChars, r) { - // whitespace or option ends the option - // => add option unsanitized - optionDetected = false - if len(buff) > 0 { - sanCmdLine += buff - buff = "" - } - sanCmdLine += string(r) - } else if unicode.IsLetter(r) == false && unicode.IsDigit(r) == false && - strings.ContainsRune(optionAllowedChars, r) == false { - // r is not any of allowed chars for an option: letter, digit, "-" or "_" - // => sanitize - if len(buff) > 0 { - sanToken, err := s.sanitizeCmdToken(buff) - if err != nil { - log.Println("WARN: got error while sanitizing cmdLine:", cmdLine) - // return cmdLine, err - } - sanCmdLine += sanToken - buff = "" - } - sanCmdLine += string(r) - } else { - buff += string(r) - } - case false: - // split command on all non-letter and non-digit characters - if unicode.IsLetter(r) == false && unicode.IsDigit(r) == false { - // split token - if len(buff) > 0 { - sanToken, err := s.sanitizeCmdToken(buff) - if err != nil { - log.Println("WARN: got error while sanitizing cmdLine:", cmdLine) - // return cmdLine, err - } - sanCmdLine += sanToken - buff = "" - } - sanCmdLine += string(r) - } else { - if (unicode.IsSpace(prevR2) && prevR == '-') || - (unicode.IsSpace(prevR3) && prevR2 == '-' && prevR == '-') { - optionDetected = true - } - buff += string(r) - } - } - prevR3 = prevR2 - prevR2 = prevR - prevR = r - } - if len(buff) <= 0 { - // nothing in the buffer => work is done - return sanCmdLine, nil - } - if optionDetected { - // option detected => dont sanitize - sanCmdLine += buff - return sanCmdLine, nil - } - // sanitize - sanToken, err := s.sanitizeCmdToken(buff) - if err != nil { - log.Println("WARN: got error while sanitizing cmdLine:", cmdLine) - // return cmdLine, err - } - sanCmdLine += sanToken - return sanCmdLine, nil -} - -func (s *sanitizer) sanitizeGitURL(rawURL string) (string, error) { - if len(rawURL) <= 0 { - return rawURL, nil - } - parsedURL, err := giturls.Parse(rawURL) - if err != nil { - return rawURL, err - } - return s.sanitizeParsedURL(parsedURL) -} - -func (s *sanitizer) sanitizeURL(rawURL string) (string, error) { - if len(rawURL) <= 0 { - return rawURL, nil - } - parsedURL, err := url.Parse(rawURL) - if err != nil { - return rawURL, err - } - return s.sanitizeParsedURL(parsedURL) -} - -func (s *sanitizer) sanitizeParsedURL(parsedURL *url.URL) (string, error) { - parsedURL.Opaque = s.sanitizeToken(parsedURL.Opaque) - - userinfo := parsedURL.User.Username() // only get username => password won't even make it to the sanitized data - if len(userinfo) > 0 { - parsedURL.User = url.User(s.sanitizeToken(userinfo)) - } else { - // we need to do this because `gitUrls.Parse()` sets `User` to `url.User("")` instead of `nil` - parsedURL.User = nil - } - var err error - parsedURL.Host, err = s.sanitizeTwoPartToken(parsedURL.Host, ":") - if err != nil { - return parsedURL.String(), err - } - parsedURL.Path = s.sanitizePath(parsedURL.Path) - // ForceQuery bool - parsedURL.RawQuery = s.sanitizeToken(parsedURL.RawQuery) - parsedURL.Fragment = s.sanitizeToken(parsedURL.Fragment) - - return parsedURL.String(), nil -} - -func (s *sanitizer) sanitizePath(path string) string { - var sanPath string - for _, token := range strings.Split(path, "/") { - if s.whitelist[token] != true { - token = s.hashToken(token) - } - sanPath += token + "/" - } - if len(sanPath) > 0 { - sanPath = sanPath[:len(sanPath)-1] - } - return sanPath -} - -func (s *sanitizer) sanitizeTwoPartToken(token string, delimeter string) (string, error) { - tokenParts := strings.Split(token, delimeter) - if len(tokenParts) <= 1 { - return s.sanitizeToken(token), nil - } - if len(tokenParts) == 2 { - return s.sanitizeToken(tokenParts[0]) + delimeter + s.sanitizeToken(tokenParts[1]), nil - } - return token, errors.New("Token has more than two parts") -} - -func (s *sanitizer) sanitizeCmdToken(token string) (string, error) { - // there shouldn't be tokens with letters or digits mixed together with symbols - if len(token) <= 1 { - // NOTE: do not sanitize single letter tokens - return token, nil - } - if s.isInWhitelist(token) == true { - return token, nil - } - - isLettersOrDigits := true - // isDigits := true - isOtherCharacters := true - for _, r := range token { - if unicode.IsDigit(r) == false && unicode.IsLetter(r) == false { - isLettersOrDigits = false - // isDigits = false - } - // if unicode.IsDigit(r) == false { - // isDigits = false - // } - if unicode.IsDigit(r) || unicode.IsLetter(r) { - isOtherCharacters = false - } - } - // NOTE: I decided that I don't want a special sanitization for numbers - // if isDigits { - // return s.hashNumericToken(token), nil - // } - if isLettersOrDigits { - return s.hashToken(token), nil - } - if isOtherCharacters { - return token, nil - } - log.Println("WARN: cmd token is made of mix of letters or digits and other characters; token:", token) - // return token, errors.New("cmd token is made of mix of letters or digits and other characters") - return s.hashToken(token), errors.New("cmd token is made of mix of letters or digits and other characters") -} - -func (s *sanitizer) sanitizeToken(token string) string { - if len(token) <= 1 { - // NOTE: do not sanitize single letter tokens - return token - } - if s.isInWhitelist(token) { - return token - } - return s.hashToken(token) -} - -func (s *sanitizer) hashToken(token string) string { - if len(token) <= 0 { - return token - } - // hash with sha256 - sum := sha256.Sum256([]byte(token)) - return s.trimHash(hex.EncodeToString(sum[:])) -} - -func (s *sanitizer) hashNumericToken(token string) string { - if len(token) <= 0 { - return token - } - sum := sha256.Sum256([]byte(token)) - sumInt := int(binary.LittleEndian.Uint64(sum[:])) - if sumInt < 0 { - return strconv.Itoa(sumInt * -1) - } - return s.trimHash(strconv.Itoa(sumInt)) -} - -func (s *sanitizer) trimHash(hash string) string { - length := s.hashLength - if length <= 0 || len(hash) < length { - length = len(hash) - } - return hash[:length] -} - -func (s *sanitizer) isInWhitelist(token string) bool { - return s.whitelist[strings.ToLower(token)] == true -} diff --git a/cmd/session-init/main.go b/cmd/session-init/main.go index 75f5298..c966f23 100644 --- a/cmd/session-init/main.go +++ b/cmd/session-init/main.go @@ -3,83 +3,41 @@ package main import ( "flag" "fmt" - "log" "os" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/collect" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" - "os/user" - "path/filepath" "strconv" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string +var developement bool func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") - - machineIDPath := "/etc/machine-id" - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) + config, errCfg := cfg.New() + logger, _ := logger.New("collect", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) } + out := output.New(logger, "resh-collect ERROR") + showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") requireVersion := flag.String("requireVersion", "", "abort if version doesn't match") requireRevision := flag.String("requireRevision", "", "abort if revision doesn't match") - shell := flag.String("shell", "", "actual shell") - uname := flag.String("uname", "", "uname") sessionID := flag.String("sessionId", "", "resh generated session id") - // posix variables - cols := flag.String("cols", "-1", "$COLUMNS") - lines := flag.String("lines", "-1", "$LINES") - home := flag.String("home", "", "$HOME") - lang := flag.String("lang", "", "$LANG") - lcAll := flag.String("lcAll", "", "$LC_ALL") - login := flag.String("login", "", "$LOGIN") - shellEnv := flag.String("shellEnv", "", "$SHELL") - term := flag.String("term", "", "$TERM") - - // non-posix - pid := flag.Int("pid", -1, "$$") - sessionPid := flag.Int("sessionPid", -1, "$$ at session start") - shlvl := flag.Int("shlvl", -1, "$SHLVL") - - host := flag.String("host", "", "$HOSTNAME") - hosttype := flag.String("hosttype", "", "$HOSTTYPE") - ostype := flag.String("ostype", "", "$OSTYPE") - machtype := flag.String("machtype", "", "$MACHTYPE") - - // before after - timezoneBefore := flag.String("timezoneBefore", "", "") - - osReleaseID := flag.String("osReleaseId", "", "/etc/os-release ID") - osReleaseVersionID := flag.String("osReleaseVersionId", "", - "/etc/os-release ID") - osReleaseIDLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") - osReleaseName := flag.String("osReleaseName", "", "/etc/os-release ID") - osReleasePrettyName := flag.String("osReleasePrettyName", "", - "/etc/os-release ID") - - rtb := flag.String("realtimeBefore", "-1", "before $EPOCHREALTIME") - rtsess := flag.String("realtimeSession", "-1", - "on session start $EPOCHREALTIME") - rtsessboot := flag.String("realtimeSessSinceBoot", "-1", - "on session start $EPOCHREALTIME") + sessionPID := flag.Int("sessionPid", -1, "$$ at session start") flag.Parse() if *showVersion == true { @@ -91,96 +49,16 @@ func main() { os.Exit(0) } if *requireVersion != "" && *requireVersion != version { - fmt.Println("Please restart/reload this terminal session " + - "(resh version: " + version + - "; resh version of this terminal session: " + *requireVersion + - ")") - os.Exit(3) + out.FatalVersionMismatch(version, *requireVersion) } if *requireRevision != "" && *requireRevision != commit { - fmt.Println("Please restart/reload this terminal session " + - "(resh revision: " + commit + - "; resh revision of this terminal session: " + *requireRevision + - ")") - os.Exit(3) - } - realtimeBefore, err := strconv.ParseFloat(*rtb, 64) - if err != nil { - log.Fatal("Flag Parsing error (rtb):", err) + // this is only relevant for dev versions so we can reuse FatalVersionMismatch() + out.FatalVersionMismatch("revision "+commit, "revision "+*requireVersion) } - realtimeSessionStart, err := strconv.ParseFloat(*rtsess, 64) - if err != nil { - log.Fatal("Flag Parsing error (rt sess):", err) - } - realtimeSessSinceBoot, err := strconv.ParseFloat(*rtsessboot, 64) - if err != nil { - log.Fatal("Flag Parsing error (rt sess boot):", err) - } - realtimeSinceSessionStart := realtimeBefore - realtimeSessionStart - realtimeSinceBoot := realtimeSessSinceBoot + realtimeSinceSessionStart - - timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(*timezoneBefore) - realtimeBeforeLocal := realtimeBefore + timezoneBeforeOffset - - if *osReleaseID == "" { - *osReleaseID = "linux" - } - if *osReleaseName == "" { - *osReleaseName = "Linux" - } - if *osReleasePrettyName == "" { - *osReleasePrettyName = "Linux" - } - - rec := records.Record{ - // posix - Cols: *cols, - Lines: *lines, - // core - BaseRecord: records.BaseRecord{ - Shell: *shell, - Uname: *uname, - SessionID: *sessionID, - - // posix - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - Pid: *pid, - SessionPID: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - - RealtimeBefore: realtimeBefore, - RealtimeBeforeLocal: realtimeBeforeLocal, - - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - MachineID: collect.ReadFileContent(machineIDPath), - - OsReleaseID: *osReleaseID, - OsReleaseVersionID: *osReleaseVersionID, - OsReleaseIDLike: *osReleaseIDLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, - ReshUUID: collect.ReadFileContent(reshUUIDPath), - ReshVersion: version, - ReshRevision: commit, - }, + rec := recordint.SessionInit{ + SessionID: *sessionID, + SessionPID: *sessionPID, } - collect.SendRecord(rec, strconv.Itoa(config.Port), "/session_init") + collect.SendSessionInit(out, rec, strconv.Itoa(config.Port)) } diff --git a/conf/config.toml b/conf/config.toml index 9db8b94..85109aa 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -1,5 +1 @@ -port = 2627 -sesswatchPeriodSeconds = 120 -sesshistInitHistorySize = 1000 -debug = false -bindControlR = true +configVersion = "v1" diff --git a/data/sanitizer/copyright_information.md b/data/sanitizer/copyright_information.md deleted file mode 100644 index abdbf33..0000000 --- a/data/sanitizer/copyright_information.md +++ /dev/null @@ -1,7 +0,0 @@ -# copyright information - -Whitelist contains content from variety of sources. - -Part of the whitelist (`./whitelist.txt`) is made of copyrighted content from [FileInfo.com](https://fileinfo.com/filetypes/common). - -This content was used with permission from FileInfo.com. diff --git a/data/sanitizer/whitelist.txt b/data/sanitizer/whitelist.txt deleted file mode 100644 index 180e9c3..0000000 --- a/data/sanitizer/whitelist.txt +++ /dev/null @@ -1,1195 +0,0 @@ - -! -- -. -.. -: -[ -[[ -]] -{ -} -3dm -3ds -3g2 -3gp -7z -accdb -add -addgnupghome -addgroup -addpart -addr2line -add-shell -adduser -agetty -ai -aif -alias -alternatives -apk -app -applydeltarpm -applygnupgdefaults -apt -apt-cache -apt-cdrom -apt-config -apt-get -apt-key -apt-mark -ar -arch -arpd -arping -as -asf -asm -asp -aspx -au -autoload -avi -awk -b -b2sum -badblocks -bak -base32 -base64 -basename -basenc -bash -bashbug -bashbug-64 -bat -bg -bin -bind -bindkey -bisect -blend -blkdeactivate -blkdiscard -blkid -blkzone -blockdev -bmp -boot -bootctl -br -branch -break -bridge -brotli -build-locale-archive -builtin -bunzip2 -busctl -bye -bz2 -bzcat -bzcmp -bzdiff -bzegrep -bzexe -bzfgrep -bzgrep -bzip2 -bzip2recover -bzless -bzmore -c -cab -cal -ca-legacy -caller -capsh -captoinfo -case -cat -catchsegv -cbr -cc -cd -cer -certutil -cfdisk -cfg -c++filt -cfm -cgi -chacl -chage -chardetect -chattr -chcon -chcpu -chdir -checkout -chfn -chgpasswd -chgrp -chkconfig -chmem -chmod -choom -chown -chpasswd -chroot -chrt -chsh -cksum -class -clear -clear_console -clock -clockdiff -clone -cmp -cmsutil -co -code -col -colcrt -colrm -column -com -combinedeltarpm -comm -command -commit -compadd -comparguments -compcall -compctl -compdescribe -compfiles -compgen -compgroups -complete -compopt -compquote -compset -comptags -comptry -compvalues -conf -continue -convert -coproc -coredumpctl -cp -cpgr -cpio -cpl -cpp -cppw -cracklib-check -cracklib-format -cracklib-packer -cracklib-unpacker -crdownload -create-cracklib-dict -crlutil -crx -cs -csplit -csr -css -csv -ctrlaltdel -ctstat -cue -cur -curl -cut -cvtsudoers -cz -dash -dat -date -db -db_archive -db_checkpoint -db_deadlock -db_dump -db_dump185 -dbf -db_hotbackup -db_load -db_log_verify -db_printlog -db_recover -db_replicate -db_stat -db_tuner -db_upgrade -dbus-binding-tool -dbus-broker -dbus-broker-launch -dbus-cleanup-sockets -dbus-daemon -dbus-monitor -dbus-run-session -dbus-send -dbus-test-tool -dbus-update-activation-environment -dbus-uuidgen -db_verify -dcr -dd -dds -de -deb -debconf -debconf-apt-progress -debconf-communicate -debconf-copydb -debconf-escape -debconf-set-selections -debconf-show -deb-systemd-helper -deb-systemd-invoke -debugfs -debuginfo-install -declare -delgroup -delpart -deluser -dem -depmod -deskthemepack -desktop -dev -devlink -df -dgawk -diff -diff3 -dir -dircolors -dirmngr -dirmngr-client -dirname -dirs -disable -disown -dll -dmesg -dmfilemapd -dmg -dmp -dmsetup -dmstats -dnf -dnf-3 -dnsdomainname -do -doc -docker -Dockerfile -docx -domainname -done -dpkg -dpkg-deb -dpkg-divert -dpkg-maintscript-helper -dpkg-preconfigure -dpkg-query -dpkg-reconfigure -dpkg-split -dpkg-statoverride -dpkg-trigger -dracut -drv -dtd -du -dumpe2fs -dwg -dwp -dxf -e2freefrag -e2fsck -e2image -e2label -e2mmpstatus -e2undo -e4crypt -e4defrag -easy_install-3.7 -echo -echotc -echoti -egrep -eject -elfedit -elif -else -emacs -emulate -enable -end -env -eps -esac -etc -eval -evmctl -ex -exe -exec -exit -expand -expiry -export -expr -factor -faillock -faillog -fallocate -false -fc -fdformat -fdisk -fetch -ffmpeg -fg -fgrep -fi -filefrag -fincore -find -findfs -findmnt -find-repos-of-install -fips-finish-install -fips-mode-setup -fish -fla -float -flock -flv -fmt -fnt -fold -fon -for -foreach -free -fsck -fsck.cramfs -fsck.ext2 -fsck.ext3 -fsck.ext4 -fsck.minix -fsfreeze -fstab-decode -fstrim -function -functions -g13 -g13-syshelp -gadget -gam -gapplication -gawk -gdbus -ged -gencat -genl -getcap -getconf -getent -getfacl -getln -getopt -getopts -getpcaps -getty -gif -gio -gio-launch-desktop -gio-querymodules-64 -git -github.com -glib-compile-schemas -glibc_post_upgrade.x86_64 -go -gpasswd -gpg -gpg2 -gpg-agent -gpgconf -gpg-connect-agent -gpg-error -gpgme-json -gpgparsemail -gpgsplit -gpgv -gpgv2 -gpg-wks-server -gpg-zip -gprof -gpx -grep -groupadd -groupdel -groupmems -groupmod -groups -grpck -grpconv -grpunconv -gsettings -gtar -gunzip -gz -gzexe -gzip -h -halt -hardlink -hash -head -heic -help -hexdump -history -home -hostid -hostname -hostnamectl -hqx -htm -html -http -https -hwclock -i386 -icns -ico -iconv -iconvconfig -iconvconfig.x86_64 -ics -id -idn -if -ifenslave -iff -igawk -in -indd -info -infocmp -infokey -infotocap -ini -init -initctl -insmod -install -install-info -installkernel -integer -invoke-rc.d -ionice -ip -ipcmk -ipcrm -ipcs -ir -ischroot -iso -isosize -it -jar -java -jobs -join -journalctl -jpg -jq -js -json -jsp -kernel-install -key -keychain -kill -killall5 -kml -kmod -kmz -kpartx -ksp -kss -kwd -last -lastb -lastlog -lchage -lchfn -lchsh -ld -ldattach -ld.bfd -ldconfig -ldconfig.real -ldd -ld.gold -let -lgroupadd -lgroupdel -lgroupmod -lib -lib64 -lid -limit -link -linux32 -linux64 -ln -lnewusers -lnk -lnstat -local -locale -locale-check -localectl -localedef -localhost -log -logger -login -loginctl -logname -logout -logsave -look -losetup -lost+found -lpasswd -ls -lsattr -lsblk -lscpu -lsinitrd -lsipc -lslocks -lslogins -lsmem -lsmod -lsns -lua -luac -luseradd -luserdel -lusermod -lz4 -lz4c -lz4cat -m -m3u -m4a -m4p -m4v -machinectl -make -makedb -makedeltarpm -make-dummy-cert -Makefile -man -mapfile -master -mawk -max -mcookie -md5 -md5sum -md5sums -md5sum.textutils -mdb -mdf -media -merge -mesg -mid -mim -mkdict -mkdir -mke2fs -mkfifo -mkfs -mkfs.bfs -mkfs.cramfs -mkfs.ext2 -mkfs.ext3 -mkfs.ext4 -mkfs.minix -mkhomedir_helper -mkinitrd -mklost+found -mknod -mkpasswd -mkswap -mktemp -mnt -mo -modinfo -modprobe -modulemd-validator-v1 -modutil -more -mount -mountpoint -mov -mp3 -mp4 -mpa -mpg -msg -msi -mv -namei -nawk -needs-restarting -nes -net -networkctl -newgidmap -newgrp -newuidmap -newusers -nice -nisdomainname -nl -nm -no -nocorrect -noglob -nohup -nologin -nproc -nsenter -nstat -numfmt -o -obj -objcopy -objdump -od -odt -ogg -oldfind -openssl -opt -org -origin -otf -p11-kit -package-cleanup -packer -pager -pages -pam-auth-update -pam_console_apply -pam_extrausers_chkpwd -pam_extrausers_update -pam_getenv -pam_tally -pam_tally2 -pam_timestamp_check -part -partx -passwd -paste -patch -pathchk -pct -pdb -pdf -perl -perl5.26.1 -perl5.28.1 -pgawk -pgrep -php -phps -phtml -pidof -pinentry -pinentry-curses -ping -ping4 -ping6 -pinky -pip-3 -pip3 -pip-3.7 -pip3.7 -pivot_root -pk12util -pkg -pkg-config -pkill -pkl -pl -pldd -pls -plugin -pmap -png -policy-rc.d -popd -portablectl -pov -poweroff -pps -ppt -pptx -pr -prf -print -printenv -printf -private -prlimit -proc -properties -ps -psd -pspimage -ptx -pull -push -pushd -pushln -pwck -pwconv -pwd -pwdx -pwhistory_helper -pwmake -pwscore -pwunconv -py -pyc -pydoc -pydoc3 -pydoc3.7 -pyo -python -python2 -python2.7 -python3 -python3.7 -python3.7m -pyvenv -pyvenv-3.7 -r -ranlib -rar -raw -rbash -rc -rdf -rdisc -rdma -read -readarray -readelf -readlink -readonly -readprofile -realpath -rebase -reboot -rehash -remove-shell -rename -rename.ul -renew-dummy-cert -renice -repeat -repoclosure -repodiff -repo-graph -repomanage -repoquery -repo-rss -reposync -repotrack -reset -resh -resize2fs -resizepart -resolvconf -resolvectl -return -rev -rfkill -rgrep -rm -rmdir -rmmod -rmt -rmt-tar -rom -root -routef -routel -rpcgen -rpm -rpm2archive -rpm2cpio -rpmdb -rpmdumpheader -rpmkeys -rpmquery -rpmverify -rss -rtacct -rtcwake -rtf -rtmon -rtstat -ru -run -runcon -run-help -runlevel -run-parts -runuser -rvi -rview -s -sasldblistusers2 -saslpasswd2 -sav -savelog -sbin -sched -script -scriptreplay -sdf -sdiff -sed -sefcontext_compile -select -select-editor -sensible-browser -sensible-editor -sensible-pager -seq -service -set -setarch -setcap -setfacl -setopt -setpriv -setsid -setterm -setup-nsssysinit -setup-nsssysinit.sh -sfdisk -sg -sh -sha1sum -sha224sum -sha256sum -sha384sum -sha512sum -shadowconfig -share -sh.distrib -shift -shopt -show -show-changed-rco -show-installed -shred -shuf -shutdown -signtool -signver -sitx -size -skill -slabtop -sleep -sln -snice -so -sort -sotruss -source -split -sprof -sql -sqlite3 -srt -srv -ss -ssh -ssltap -start-stop-daemon -stat -status -stdbuf -strings -strip -stty -su -sudo -sudoedit -sudoreplay -sulogin -sum -suspend -svg -swaplabel -swapoff -swapon -swf -swift -switch_root -sync -sys -sysctl -systemctl -systemd-analyze -systemd-ask-password -systemd-cat -systemd-cgls -systemd-cgtop -systemd-coredumpctl -systemd-delta -systemd-detect-virt -systemd-escape -systemd-firstboot -systemd-hwdb -systemd-id128 -systemd-inhibit -systemd-loginctl -systemd-machine-id-setup -systemd-mount -systemd-notify -systemd-nspawn -systemd-path -systemd-resolve -systemd-run -systemd-socket-activate -systemd-stdio-bridge -systemd-sysusers -systemd-tmpfiles -systemd-tty-ask-password-agent -systemd-umount -tabs -tac -tag -tail -tailf -tar -tarcat -taskset -tax2016 -tax2018 -tc -tee -telinit -tempfile -test -testgdbm -tex -tga -tgz -then -thm -tic -tif -tiff -tig -time -timedatectl -timeout -times -tipc -tload -tmp -toast -toe -top -torrent -touch -tput -tr -tracepath -tracepath6 -trap -true -truncate -trust -tset -tsort -ttf -tty -ttyctl -tune2fs -txt -type -typeset -tzconfig -tzselect -udevadm -uk -ul -ulimit -umask -umount -unalias -uname -uname26 -unbound-anchor -uncompress -unexpand -unfunction -unhash -uniq -unix_chkpwd -unix_update -unlimit -unlink -unlz4 -unminimize -unset -unsetopt -unshare -until -unxz -update-alternatives -update-ca-trust -update-crypto-policies -update-mime-database -update-passwd -update-rc.d -uptime -urlgrabber -useradd -userdel -usermod -users -usr -utmpdump -uue -uuidgen -uuidparse -Vagrantfile -var -vared -vb -vcd -vcf -vcxproj -vdir -verifytree -vi -view -vigr -vim -vipw -visudo -vlc -vmstat -vob -w -wait -wall -watch -watchgnupg -wav -wc -wdctl -weak-modules -whence -where -whereis -which -which-command -while -who -whoami -wipefs -wma -wmv -wpd -w.procps -wps -write -wsf -x86_64 -xargs -xbel -xcodeproj -xhtml -xlr -xls -xlsx -xml -xmlcatalog -xmllint -xmlwf -xpm -xsd -xsl -xz -xzcat -xzcmp -xzdec -xzdiff -xzegrep -xzfgrep -xzgrep -xzless -xzmore -yaourt -yes -ypdomainname -yum -yum-builddep -yum-complete-transaction -yum-config-manager -yumdb -yum-debug-dump -yum-debug-restore -yumdownloader -yum-groups-manager -yuv -Z -zcat -zcmp -zcompile -zdiff -zdump -zegrep -zfgrep -zforce -zformat -zgrep -zic -zip -zipx -zle -zless -zmodload -zmore -znew -zparseopts -zramctl -zregexparse -zsh -zstyle diff --git a/go.mod b/go.mod index 0ca87cd..db1e19f 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,27 @@ module github.com/curusarn/resh -go 1.16 +go 1.18 require ( github.com/BurntSushi/toml v0.4.1 github.com/awesome-gocui/gocui v1.0.0 - github.com/coreos/go-semver v0.3.0 - github.com/gdamore/tcell/v2 v2.4.0 // indirect - github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 - github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mattn/go-shellwords v1.0.12 - github.com/mb-14/gomarkov v0.0.0-20210216094942-a5b484cc0243 github.com/mitchellh/go-ps v1.0.0 - github.com/schollz/progressbar v1.0.0 github.com/spf13/cobra v1.2.1 github.com/whilp/git-urls v1.0.0 + go.uber.org/zap v1.21.0 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 +) + +require ( + github.com/gdamore/encoding v1.0.0 // indirect + github.com/gdamore/tcell/v2 v2.4.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect golang.org/x/sys v0.0.0-20210903071746-97244b99971b // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index 60670d5..341fecd 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awesome-gocui/gocui v1.0.0 h1:1bf0DAr2JqWNxGFS8Kex4fM/khICjEnCi+a1+NfWy+w= github.com/awesome-gocui/gocui v1.0.0/go.mod h1:UvP3dP6+UsTGl9IuqP36wzz6Lemo90wn5p3tJvZ2OqY= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -57,7 +58,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -170,8 +170,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 h1:1dSBUfGlorLAua2CRx0zFN7kQsTpE2DQSmr7rrTNgY8= -github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629/go.mod h1:mb5nS4uRANwOJSZj8rlCWAfAcGi72GGMIXx+xGOjA7M= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -193,10 +191,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mb-14/gomarkov v0.0.0-20210216094942-a5b484cc0243 h1:F0IAcxxFNzC8/HOxI5Q2hpsWAoGdy+lGMjoVyrcMeSw= -github.com/mb-14/gomarkov v0.0.0-20210216094942-a5b484cc0243/go.mod h1:5F3Y03oxWIyMq3Wa4AxU544RYnXNZwHBfqpDpdLibBY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -211,7 +205,6 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -226,8 +219,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= -github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -265,9 +256,14 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -412,7 +408,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b h1:3Dq0eVHn0uaQJmPO+/aYPI/fRMqdrVDbu7MQcku54gg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -482,6 +477,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -595,7 +591,6 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go new file mode 100644 index 0000000..25a0a8f --- /dev/null +++ b/internal/cfg/cfg.go @@ -0,0 +1,227 @@ +package cfg + +import ( + "fmt" + "os" + "path" + + "github.com/BurntSushi/toml" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// configFile used to parse the config file +type configFile struct { + // ConfigVersion - never remove this + ConfigVersion *string + + // added in legacy + Port *int + SesswatchPeriodSeconds *uint + SesshistInitHistorySize *int + BindControlR *bool + Debug *bool + + // added in v1 + LogLevel *string + + // added in legacy + // deprecated in v1 + BindArrowKeysBash *bool + BindArrowKeysZsh *bool + + SyncConnectorAddress *string + SyncConnectorAuthToken *string + SyncConnectorPullPeriodSeconds *int + SyncConnectorSendPeriodSeconds *int +} + +// Config returned by this package to be used in the rest of the project +type Config struct { + // Port used by daemon and rest of the components to communicate + // Make sure to restart the daemon when you change it + Port int + + // BindControlR causes CTRL+R to launch the search app + BindControlR bool + // LogLevel used to filter logs + LogLevel zapcore.Level + + // Debug mode for search app + Debug bool + // SessionWatchPeriodSeconds is how often should daemon check if terminal + // sessions are still alive + // There is not much need to adjust the value both memory overhead of watched sessions + // and the CPU overhead of checking them are relatively low + SessionWatchPeriodSeconds uint + // ReshHistoryMinSize is how large resh history needs to be for + // daemon to ignore standard shell history files + // Ignoring standard shell history gives us more consistent experience, + // but you can increase this to something large to see standard shell history in RESH search + ReshHistoryMinSize int + + // SyncConnectorAddress used by the daemon to connect to the Sync Connector + // examples: + // - http://localhost:1234 + // - http://localhost:1234 + // - http://192.168.1.1:1324 + // - https://domain.tld + // - https://domain.tld/resh + SyncConnectorAddress *string + + // SyncConnectorAuthToken used by the daemon to authenticate with the Sync Connector + SyncConnectorAuthToken string + + // SyncConnectorPullPeriodSeconds how often should Resh daemon download history from Sync Connector + SyncConnectorPullPeriodSeconds int + + // SyncConnectorSendPeriodSeconds how often should Resh daemon send history to the Sync Connector + SyncConnectorSendPeriodSeconds int +} + +// defaults for config +var defaults = Config{ + Port: 2627, + LogLevel: zap.InfoLevel, + BindControlR: true, + + Debug: false, + SessionWatchPeriodSeconds: 600, + ReshHistoryMinSize: 1000, + + SyncConnectorPullPeriodSeconds: 30, + SyncConnectorSendPeriodSeconds: 30, +} + +const headerComment = `## +###################### +## RESH config (v1) ## +###################### +## Here you can find info about RESH configuration options. +## You can uncomment the options and custimize them. + +## Required. +## The config format can change in future versions. +## ConfigVersion helps us seemlessly upgrade to the new formats. +# ConfigVersion = "v1" + +## Port used by RESH daemon and rest of the components to communicate. +## Make sure to restart the daemon (pkill resh-daemon) when you change it. +# Port = 2627 + +## Controls how much and how detailed logs all RESH components produce. +## Use "debug" for full logs when you encounter an issue +## Options: "debug", "info", "warn", "error", "fatal" +# LogLevel = "info" + +## When BindControlR is "true" RESH search app is bound to CTRL+R on terminal startuA +# BindControlR = true + +## When Debug is "true" the RESH search app runs in debug mode. +## This is useful for development. +# Debug = false + +## Daemon keeps track of running terminal sessions. +## SessionWatchPeriodSeconds controls how often daemon checks if the sessions are still alive. +## You shouldn't need to adjust this. +# SessionWatchPeriodSeconds = 600 + +## When RESH is first installed there is no RESH history so there is nothing to search. +## As a temporary woraround, RESH daemon parses bash/zsh shell history and searches it. +## Once RESH history is big enough RESH stops using bash/zsh history. +## ReshHistoryMinSize controls how big RESH history needs to be before this happens. +## You can increase this this to e.g. 10000 to get RESH to use bash/zsh history longer. +# ReshHistoryMinSize = 1000 + +` + +// TODO: Add description for the new options. + +func getConfigPath() (string, error) { + fname := "resh.toml" + xdgDir, found := os.LookupEnv("XDG_CONFIG_HOME") + if found { + return path.Join(xdgDir, fname), nil + } + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("could not get user home dir: %w", err) + } + return path.Join(homeDir, ".config", fname), nil +} + +func readConfig(path string) (*configFile, error) { + var config configFile + if _, err := toml.DecodeFile(path, &config); err != nil { + return &config, fmt.Errorf("could not decode config: %w", err) + } + return &config, nil +} + +func getConfig() (*configFile, error) { + path, err := getConfigPath() + if err != nil { + return nil, fmt.Errorf("could not get config file path: %w", err) + } + return readConfig(path) +} + +// returned config is always usable, returned errors are informative +func processAndFillDefaults(configF *configFile) (Config, error) { + config := defaults + + if configF.Port != nil { + config.Port = *configF.Port + } + if configF.SesswatchPeriodSeconds != nil { + config.SessionWatchPeriodSeconds = *configF.SesswatchPeriodSeconds + } + if configF.SesshistInitHistorySize != nil { + config.ReshHistoryMinSize = *configF.SesshistInitHistorySize + } + + var err error + if configF.LogLevel != nil { + logLevel, err := zapcore.ParseLevel(*configF.LogLevel) + if err != nil { + err = fmt.Errorf("could not parse log level: %w", err) + } else { + config.LogLevel = logLevel + } + } + + if configF.BindControlR != nil { + config.BindControlR = *configF.BindControlR + } + + config.SyncConnectorAddress = configF.SyncConnectorAddress + + if configF.SyncConnectorAuthToken != nil { + config.SyncConnectorAuthToken = *configF.SyncConnectorAuthToken + } + + if configF.SyncConnectorPullPeriodSeconds != nil { + config.SyncConnectorPullPeriodSeconds = *configF.SyncConnectorPullPeriodSeconds + } + + if configF.SyncConnectorSendPeriodSeconds != nil { + config.SyncConnectorSendPeriodSeconds = *configF.SyncConnectorSendPeriodSeconds + } + + return config, err +} + +// New returns a config file +// returned config is always usable, returned errors are informative +func New() (Config, error) { + configF, err := getConfig() + if err != nil { + return defaults, fmt.Errorf("using default config because of error while getting/reading config: %w", err) + } + + config, err := processAndFillDefaults(configF) + if err != nil { + return config, fmt.Errorf("errors while processing config: %w", err) + } + return config, nil +} diff --git a/internal/cfg/migrate.go b/internal/cfg/migrate.go new file mode 100644 index 0000000..fbbb214 --- /dev/null +++ b/internal/cfg/migrate.go @@ -0,0 +1,117 @@ +package cfg + +import ( + "fmt" + "os" + + "github.com/BurntSushi/toml" +) + +// Touch config file +func Touch() error { + path, err := getConfigPath() + if err != nil { + return fmt.Errorf("could not get config file path: %w", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) + if err != nil { + return fmt.Errorf("could not open/create config file: %w", err) + } + err = file.Close() + if err != nil { + return fmt.Errorf("could not close config file: %w", err) + } + return nil +} + +// Migrate old config versions to current config version +// returns true if any changes were made to the config +func Migrate() (bool, error) { + path, err := getConfigPath() + if err != nil { + return false, fmt.Errorf("could not get config file path: %w", err) + } + configF, err := readConfig(path) + if err != nil { + return false, fmt.Errorf("could not read config: %w", err) + } + const current = "v1" + if configF.ConfigVersion != nil && *configF.ConfigVersion == current { + return false, nil + } + + if configF.ConfigVersion == nil { + configF, err = legacyToV1(configF) + if err != nil { + return true, fmt.Errorf("error converting config from version 'legacy' to 'v1': %w", err) + } + } + + if *configF.ConfigVersion != current { + return false, fmt.Errorf("unrecognized config version: '%s'", *configF.ConfigVersion) + } + err = writeConfig(configF, path) + if err != nil { + return true, fmt.Errorf("could not write migrated config: %w", err) + } + return true, nil +} + +// writeConfig should only be used when migrating config to new version +// writing the config file discards all comments in the config file (limitation of TOML library) +// to make up for lost comments we add header comment to the start of the file +func writeConfig(config *configFile, path string) error { + file, err := os.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0666) + if err != nil { + return fmt.Errorf("could not open config for writing: %w", err) + } + defer file.Close() + _, err = file.WriteString(headerComment) + if err != nil { + return fmt.Errorf("could not write config header: %w", err) + } + err = toml.NewEncoder(file).Encode(config) + if err != nil { + return fmt.Errorf("could not encode config: %w", err) + } + return nil +} + +func legacyToV1(config *configFile) (*configFile, error) { + if config.ConfigVersion != nil { + return nil, fmt.Errorf("config version is not 'legacy': '%s'", *config.ConfigVersion) + } + version := "v1" + newConf := configFile{ + ConfigVersion: &version, + } + // Remove defaults + if config.Port != nil && *config.Port != 2627 { + newConf.Port = config.Port + } + if config.SesswatchPeriodSeconds != nil && *config.SesswatchPeriodSeconds != 120 { + newConf.SesswatchPeriodSeconds = config.SesswatchPeriodSeconds + } + if config.SesshistInitHistorySize != nil && *config.SesshistInitHistorySize != 1000 { + newConf.SesshistInitHistorySize = config.SesshistInitHistorySize + } + if config.BindControlR != nil && *config.BindControlR != true { + newConf.BindControlR = config.BindControlR + } + if config.Debug != nil && *config.Debug != false { + newConf.Debug = config.Debug + } + return &newConf, nil +} + +// func v1ToV2(config *configFile) (*configFile, error) { +// if *config.ConfigVersion != "v1" { +// return nil, fmt.Errorf("config version is not 'legacy': '%s'", *config.ConfigVersion) +// } +// version := "v2" +// newConf := configFile{ +// ConfigVersion: &version, +// // Here goes all config fields - no need to prune defaults like we do for legacy +// } +// return &newConf, nil +// } diff --git a/internal/collect/collect.go b/internal/collect/collect.go new file mode 100644 index 0000000..15d2bbb --- /dev/null +++ b/internal/collect/collect.go @@ -0,0 +1,116 @@ +package collect + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" +) + +// SendRecord to daemon +func SendRecord(out *output.Output, r recordint.Collect, port, path string) { + out.Logger.Debug("Sending record ...", + zap.String("cmdLine", r.Rec.CmdLine), + zap.String("sessionID", r.SessionID), + ) + recJSON, err := json.Marshal(r) + if err != nil { + out.Fatal("Error while encoding record", err) + } + + req, err := http.NewRequest("POST", "http://localhost:"+port+path, + bytes.NewBuffer(recJSON)) + if err != nil { + out.Fatal("Error while sending record", err) + } + req.Header.Set("Content-Type", "application/json") + + client := http.Client{ + Timeout: 1 * time.Second, + } + _, err = client.Do(req) + if err != nil { + out.FatalDaemonNotRunning(err) + } +} + +// SendSessionInit to daemon +func SendSessionInit(out *output.Output, r recordint.SessionInit, port string) { + out.Logger.Debug("Sending session init ...", + zap.String("sessionID", r.SessionID), + zap.Int("sessionPID", r.SessionPID), + ) + recJSON, err := json.Marshal(r) + if err != nil { + out.Fatal("Error while encoding record", err) + } + + req, err := http.NewRequest("POST", "http://localhost:"+port+"/session_init", + bytes.NewBuffer(recJSON)) + if err != nil { + out.Fatal("Error while sending record", err) + } + req.Header.Set("Content-Type", "application/json") + + client := http.Client{ + Timeout: 1 * time.Second, + } + _, err = client.Do(req) + if err != nil { + out.FatalDaemonNotRunning(err) + } +} + +// ReadFileContent and return it as a string +func ReadFileContent(logger *zap.Logger, path string) string { + dat, err := ioutil.ReadFile(path) + if err != nil { + logger.Error("Error reading file", + zap.String("filePath", path), + zap.Error(err), + ) + return "" + } + return strings.TrimSuffix(string(dat), "\n") +} + +// GetGitDirs based on result of git "cdup" command +func GetGitDirs(logger *zap.Logger, cdup string, exitCode int, pwd string) (string, string) { + if exitCode != 0 { + return "", "" + } + abspath := filepath.Clean(filepath.Join(pwd, cdup)) + realpath, err := filepath.EvalSymlinks(abspath) + if err != nil { + logger.Error("Error while handling git dir paths", zap.Error(err)) + return "", "" + } + return abspath, realpath +} + +// GetTimezoneOffsetInSeconds based on zone returned by date command +func GetTimezoneOffsetInSeconds(logger *zap.Logger, zone string) float64 { + // date +%z -> "+0200" + hoursStr := zone[:3] + minsStr := zone[3:] + hours, err := strconv.Atoi(hoursStr) + if err != nil { + logger.Error("Error while parsing hours in timezone offset", zap.Error(err)) + return -1 + } + mins, err := strconv.Atoi(minsStr) + if err != nil { + logger.Error("Errot while parsing minutes in timezone offset:", zap.Error(err)) + return -1 + } + secs := ((hours * 60) + mins) * 60 + return float64(secs) +} diff --git a/internal/datadir/datadir.go b/internal/datadir/datadir.go new file mode 100644 index 0000000..59011dd --- /dev/null +++ b/internal/datadir/datadir.go @@ -0,0 +1,71 @@ +package datadir + +import ( + "fmt" + "os" + "path" +) + +// You should not need this caching +// It messes with proper dependency injection +// Find another way + +// type dirCache struct { +// dir string +// err error +// +// cached bool +// } +// +// var cache dirCache +// +// func getPathNoCache() (string, error) { +// reshDir := "resh" +// xdgDir, found := os.LookupEnv("XDG_DATA_HOME") +// if found { +// return path.Join(xdgDir, reshDir), nil +// } +// homeDir, err := os.UserHomeDir() +// if err != nil { +// return "", fmt.Errorf("error while getting home dir: %w", err) +// } +// return path.Join(homeDir, ".local/share/", reshDir), nil +// } +// +// func GetPath() (string, error) { +// if !cache.cached { +// dir, err := getPathNoCache() +// cache = dirCache{ +// dir: dir, +// err: err, +// cached: true, +// } +// } +// return cache.dir, cache.err +// } + +func GetPath() (string, error) { + reshDir := "resh" + xdgDir, found := os.LookupEnv("XDG_DATA_HOME") + if found { + return path.Join(xdgDir, reshDir), nil + } + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("error while getting home dir: %w", err) + } + return path.Join(homeDir, ".local/share/", reshDir), nil +} + +func MakePath() (string, error) { + path, err := GetPath() + if err != nil { + return "", err + } + err = os.MkdirAll(path, 0755) + // skip "exists" error + if err != nil && !os.IsExist(err) { + return "", fmt.Errorf("error while creating directories: %w", err) + } + return path, nil +} diff --git a/internal/device/device.go b/internal/device/device.go new file mode 100644 index 0000000..8a34ead --- /dev/null +++ b/internal/device/device.go @@ -0,0 +1,49 @@ +package device + +import ( + "fmt" + "os" + "path" + "strings" +) + +func GetID(dataDir string) (string, error) { + fname := "device-id" + dat, err := os.ReadFile(path.Join(dataDir, fname)) + if err != nil { + return "", fmt.Errorf("could not read file with device-id: %w", err) + } + id := strings.TrimRight(string(dat), "\n") + return id, nil +} + +func GetName(dataDir string) (string, error) { + fname := "device-name" + dat, err := os.ReadFile(path.Join(dataDir, fname)) + if err != nil { + return "", fmt.Errorf("could not read file with device-name: %w", err) + } + name := strings.TrimRight(string(dat), "\n") + return name, nil +} + +// TODO: implement, possibly with a better name +// func CheckID(dataDir string) (string, error) { +// fname := "device-id" +// dat, err := os.ReadFile(path.Join(dataDir, fname)) +// if err != nil { +// return "", fmt.Errorf("could not read file with device-id: %w", err) +// } +// id := strings.TrimRight(string(dat), "\n") +// return id, nil +// } +// +// func CheckName(dataDir string) (string, error) { +// fname := "device-id" +// dat, err := os.ReadFile(path.Join(dataDir, fname)) +// if err != nil { +// return "", fmt.Errorf("could not read file with device-id: %w", err) +// } +// id := strings.TrimRight(string(dat), "\n") +// return id, nil +// } diff --git a/internal/histcli/histcli.go b/internal/histcli/histcli.go new file mode 100644 index 0000000..6f889a8 --- /dev/null +++ b/internal/histcli/histcli.go @@ -0,0 +1,86 @@ +package histcli + +import ( + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" + "go.uber.org/zap" + "sync" +) + +// Histcli is a dump of history preprocessed for resh cli purposes +type Histcli struct { + // list of records + list []recordint.SearchApp + // TODO It is not optimal to keep both raw and list but is necessary for syncConnector now + raw []record.V1 + knownIds map[string]struct{} + lock sync.RWMutex + sugar *zap.SugaredLogger + latest map[string]float64 +} + +// New Histcli +func New(sugar *zap.SugaredLogger) *Histcli { + return &Histcli{ + sugar: sugar.With(zap.String("component", "histCli")), + knownIds: map[string]struct{}{}, + latest: map[string]float64{}, + } +} + +// AddRecord to the histcli +func (h *Histcli) AddRecord(rec *recordint.Indexed) { + cli := recordint.NewSearchApp(rec) + h.lock.Lock() + defer h.lock.Unlock() + + if _, ok := h.knownIds[rec.Rec.RecordID]; !ok { + h.knownIds[rec.Rec.RecordID] = struct{}{} + h.list = append(h.list, cli) + h.raw = append(h.raw, rec.Rec) + h.updateLatestPerDevice(cli) + } else { + h.sugar.Debugw("Record is already present", "id", rec.Rec.RecordID) + } +} + +// AddCmdLine to the histcli +func (h *Histcli) AddCmdLine(cmdline string) { + cli := recordint.NewSearchAppFromCmdLine(cmdline) + h.lock.Lock() + defer h.lock.Unlock() + + h.list = append(h.list, cli) +} + +func (h *Histcli) Dump() []recordint.SearchApp { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.list +} + +func (h *Histcli) DumpRaw() []record.V1 { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.raw +} + +// updateLatestPerDevice should be called only with write lock because it does not lock on its own. +func (h *Histcli) updateLatestPerDevice(rec recordint.SearchApp) { + if l, ok := h.latest[rec.DeviceID]; ok { + if rec.Time > l { + h.latest[rec.DeviceID] = rec.Time + } + } else { + h.latest[rec.DeviceID] = rec.Time + } +} + +func (h *Histcli) LatestRecordsPerDevice() map[string]float64 { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.latest +} diff --git a/internal/histfile/histfile.go b/internal/histfile/histfile.go new file mode 100644 index 0000000..032a398 --- /dev/null +++ b/internal/histfile/histfile.go @@ -0,0 +1,277 @@ +package histfile + +import ( + "math" + "os" + "strconv" + "sync" + + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/histlist" + "github.com/curusarn/resh/internal/recio" + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recutil" + "github.com/curusarn/resh/record" + "go.uber.org/zap" +) + +// TODO: get rid of histfile - use histio instead +// Histfile writes records to histfile +type Histfile struct { + sugar *zap.SugaredLogger + + sessionsMutex sync.Mutex + sessions map[string]recordint.Collect + historyPath string + + // NOTE: we have separate histories which only differ if there was not enough resh_history + // resh_history itself is common for both bash and zsh + bashCmdLines histlist.Histlist + zshCmdLines histlist.Histlist + + cliRecords *histcli.Histcli + + rio *recio.RecIO +} + +// New creates new histfile and runs its gorutines +func New(sugar *zap.SugaredLogger, input chan recordint.Collect, sessionsToDrop chan string, + reshHistoryPath string, bashHistoryPath string, zshHistoryPath string, + maxInitHistSize int, minInitHistSizeKB int, + signals chan os.Signal, shutdownDone chan string, histCli *histcli.Histcli) *Histfile { + + rio := recio.New(sugar.With("module", "histfile")) + hf := Histfile{ + sugar: sugar.With("module", "histfile"), + sessions: map[string]recordint.Collect{}, + historyPath: reshHistoryPath, + bashCmdLines: histlist.New(sugar), + zshCmdLines: histlist.New(sugar), + cliRecords: histCli, + rio: &rio, + } + go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB) + go hf.writer(input, signals, shutdownDone) + go hf.sessionGC(sessionsToDrop) + return &hf +} + +// load records from resh history, reverse, enrich and save +func (h *Histfile) loadCliRecords(recs []recordint.Indexed) { + for _, cmdline := range h.bashCmdLines.List { + h.cliRecords.AddCmdLine(cmdline) + } + for _, cmdline := range h.zshCmdLines.List { + h.cliRecords.AddCmdLine(cmdline) + } + for i := len(recs) - 1; i >= 0; i-- { + rec := recs[i] + h.cliRecords.AddRecord(&rec) + } + h.sugar.Infow("Resh history loaded", + "historyRecordsCount", len(h.cliRecords.Dump()), + ) +} + +// loadsHistory from resh_history and if there is not enough of it also load native shell histories +func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHistSize, minInitHistSizeKB int) { + h.sugar.Infow("Checking if resh_history is large enough ...") + fi, err := os.Stat(h.historyPath) + var size int + if err != nil { + h.sugar.Errorw("Failed to stat resh_history file", "error", err) + } else { + size = int(fi.Size()) + } + useNativeHistories := false + if size/1024 < minInitHistSizeKB { + useNativeHistories = true + h.sugar.Warnw("Resh_history is too small - loading native bash and zsh history ...") + h.bashCmdLines = records.LoadCmdLinesFromBashFile(h.sugar, bashHistoryPath) + h.sugar.Infow("Bash history loaded", "cmdLineCount", len(h.bashCmdLines.List)) + h.zshCmdLines = records.LoadCmdLinesFromZshFile(h.sugar, zshHistoryPath) + h.sugar.Infow("Zsh history loaded", "cmdLineCount", len(h.zshCmdLines.List)) + // no maxInitHistSize when using native histories + maxInitHistSize = math.MaxInt32 + } + h.sugar.Debugw("Loading resh history from file ...", + "historyFile", h.historyPath, + ) + history, err := h.rio.ReadAndFixFile(h.historyPath, 3) + if err != nil { + h.sugar.Panicf("Failed to read file: %w", err) + } + h.sugar.Infow("Resh history loaded from file", + "historyFile", h.historyPath, + "recordCount", len(history), + ) + go h.loadCliRecords(history) + // NOTE: keeping this weird interface for now because we might use it in the future + // when we only load bash or zsh history + reshCmdLines := loadCmdLines(h.sugar, history) + h.sugar.Infow("Resh history loaded and processed", + "recordCount", len(reshCmdLines.List), + ) + if useNativeHistories == false { + h.bashCmdLines = reshCmdLines + h.zshCmdLines = histlist.Copy(reshCmdLines) + return + } + h.bashCmdLines.AddHistlist(reshCmdLines) + h.sugar.Infow("Processed bash history and resh history together", "cmdLinecount", len(h.bashCmdLines.List)) + h.zshCmdLines.AddHistlist(reshCmdLines) + h.sugar.Infow("Processed zsh history and resh history together", "cmdLineCount", len(h.zshCmdLines.List)) +} + +// sessionGC reads sessionIDs from channel and deletes them from histfile struct +func (h *Histfile) sessionGC(sessionsToDrop chan string) { + for { + func() { + session := <-sessionsToDrop + sugar := h.sugar.With("sessionID", session) + sugar.Debugw("Got session to drop") + h.sessionsMutex.Lock() + defer h.sessionsMutex.Unlock() + if part1, found := h.sessions[session]; found == true { + sugar.Infow("Dropping session") + delete(h.sessions, session) + go h.rio.AppendToFile(h.historyPath, []record.V1{part1.Rec}) + } else { + sugar.Infow("No hanging parts for session - nothing to drop") + } + }() + } +} + +// writer reads records from channel, merges them and writes them to file +func (h *Histfile) writer(collect chan recordint.Collect, signals chan os.Signal, shutdownDone chan string) { + for { + func() { + select { + case rec := <-collect: + part := "2" + if rec.Rec.PartOne { + part = "1" + } + sugar := h.sugar.With( + "recordCmdLine", rec.Rec.CmdLine, + "recordPart", part, + "recordShell", rec.Shell, + ) + sugar.Debugw("Got record") + h.sessionsMutex.Lock() + defer h.sessionsMutex.Unlock() + + // allows nested sessions to merge records properly + mergeID := rec.SessionID + "_" + strconv.Itoa(rec.Shlvl) + sugar = sugar.With("mergeID", mergeID) + if rec.Rec.PartOne { + if _, found := h.sessions[mergeID]; found { + msg := "Got another first part of the records before merging the previous one - overwriting!" + if rec.Shell == "zsh" { + sugar.Warnw(msg) + } else { + sugar.Infow(msg + " Unfortunately this is normal in bash, it can't be prevented.") + } + } + h.sessions[mergeID] = rec + } else { + if part1, found := h.sessions[mergeID]; found == false { + sugar.Warnw("Got second part of record and nothing to merge it with - ignoring!") + } else { + delete(h.sessions, mergeID) + go h.mergeAndWriteRecord(sugar, part1, rec) + } + } + case sig := <-signals: + sugar := h.sugar.With( + "signal", sig.String(), + ) + sugar.Infow("Got signal") + h.sessionsMutex.Lock() + defer h.sessionsMutex.Unlock() + sugar.Debugw("Unlocked mutex") + + for sessID, rec := range h.sessions { + sugar.Warnw("Writing incomplete record for session", + "sessionID", sessID, + ) + h.writeRecord(sugar, rec.Rec) + } + sugar.Debugw("Shutdown successful") + shutdownDone <- "histfile" + return + } + }() + } +} + +func (h *Histfile) writeRecord(sugar *zap.SugaredLogger, rec record.V1) { + h.rio.AppendToFile(h.historyPath, []record.V1{rec}) +} + +func (h *Histfile) mergeAndWriteRecord(sugar *zap.SugaredLogger, part1 recordint.Collect, part2 recordint.Collect) { + rec, err := recutil.Merge(&part1, &part2) + if err != nil { + sugar.Errorw("Error while merging records", "error", err) + return + } + + cmdLine := rec.CmdLine + h.bashCmdLines.AddCmdLine(cmdLine) + h.zshCmdLines.AddCmdLine(cmdLine) + h.cliRecords.AddRecord(&recordint.Indexed{ + // TODO: is this what we want? + Rec: rec, + }) + + h.rio.AppendToFile(h.historyPath, []record.V1{rec}) +} + +// TODO: use errors in RecIO +// func writeRecord(sugar *zap.SugaredLogger, rec record.V1, outputPath string) { +// recJSON, err := json.Marshal(rec) +// if err != nil { +// sugar.Errorw("Marshalling error", "error", err) +// return +// } +// f, err := os.OpenFile(outputPath, +// os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) +// if err != nil { +// sugar.Errorw("Could not open file", "error", err) +// return +// } +// defer f.Close() +// _, err = f.Write(append(recJSON, []byte("\n")...)) +// if err != nil { +// sugar.Errorw("Error while writing record", +// "recordRaw", rec, +// "error", err, +// ) +// return +// } +// } + +func loadCmdLines(sugar *zap.SugaredLogger, recs []recordint.Indexed) histlist.Histlist { + hl := histlist.New(sugar) + // go from bottom and deduplicate + var cmdLines []string + cmdLinesSet := map[string]bool{} + for i := len(recs) - 1; i >= 0; i-- { + cmdLine := recs[i].Rec.CmdLine + if cmdLinesSet[cmdLine] { + continue + } + cmdLinesSet[cmdLine] = true + cmdLines = append([]string{cmdLine}, cmdLines...) + // if len(cmdLines) > limit { + // break + // } + } + // add everything to histlist + for _, cmdLine := range cmdLines { + hl.AddCmdLine(cmdLine) + } + return hl +} diff --git a/internal/histio/file.go b/internal/histio/file.go new file mode 100644 index 0000000..5233717 --- /dev/null +++ b/internal/histio/file.go @@ -0,0 +1,56 @@ +package histio + +import ( + "fmt" + "os" + "sync" + + "github.com/curusarn/resh/internal/recio" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" +) + +type histfile struct { + sugar *zap.SugaredLogger + // deviceID string + path string + + mu sync.RWMutex + data []recordint.Indexed + fileinfo os.FileInfo +} + +func newHistfile(sugar *zap.SugaredLogger, path string) *histfile { + return &histfile{ + sugar: sugar.With( + // FIXME: drop V1 once original histfile is gone + "component", "histfileV1", + "path", path, + ), + // deviceID: deviceID, + path: path, + } +} + +func (h *histfile) updateFromFile() error { + rio := recio.New(h.sugar) + // TODO: decide and handle errors + newData, _, err := rio.ReadFile(h.path) + if err != nil { + return fmt.Errorf("could not read history file: %w", err) + } + h.mu.Lock() + defer h.mu.Unlock() + h.data = newData + h.updateFileInfo() + return nil +} + +func (h *histfile) updateFileInfo() error { + info, err := os.Stat(h.path) + if err != nil { + return fmt.Errorf("history file not found: %w", err) + } + h.fileinfo = info + return nil +} diff --git a/internal/histio/histio.go b/internal/histio/histio.go new file mode 100644 index 0000000..b8486de --- /dev/null +++ b/internal/histio/histio.go @@ -0,0 +1,44 @@ +package histio + +import ( + "path" + + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" + "go.uber.org/zap" +) + +type Histio struct { + sugar *zap.SugaredLogger + histDir string + + thisDeviceID string + thisHistory *histfile + // TODO: remote histories + // moreHistories map[string]*histfile + + recordsToAppend chan record.V1 + recordsToFlag chan recordint.Flag +} + +func New(sugar *zap.SugaredLogger, dataDir, deviceID string) *Histio { + sugarHistio := sugar.With(zap.String("component", "histio")) + histDir := path.Join(dataDir, "history") + currPath := path.Join(histDir, deviceID) + // TODO: file extension for the history, yes or no? (.reshjson vs. ) + + // TODO: discover other history files, exclude current + + return &Histio{ + sugar: sugarHistio, + histDir: histDir, + + thisDeviceID: deviceID, + thisHistory: newHistfile(sugar, currPath), + // moreHistories: ... + } +} + +func (h *Histio) Append(r *record.V1) { + +} diff --git a/pkg/histlist/histlist.go b/internal/histlist/histlist.go similarity index 61% rename from pkg/histlist/histlist.go rename to internal/histlist/histlist.go index a3f4334..94376f8 100644 --- a/pkg/histlist/histlist.go +++ b/internal/histlist/histlist.go @@ -1,9 +1,11 @@ package histlist -import "log" +import "go.uber.org/zap" // Histlist is a deduplicated list of cmdLines type Histlist struct { + // TODO: I'm not excited about logger being passed here + sugar *zap.SugaredLogger // list of commands lines (deduplicated) List []string // lookup: cmdLine -> last index @@ -11,13 +13,16 @@ type Histlist struct { } // New Histlist -func New() Histlist { - return Histlist{LastIndex: make(map[string]int)} +func New(sugar *zap.SugaredLogger) Histlist { + return Histlist{ + sugar: sugar.With("component", "histlist"), + LastIndex: make(map[string]int), + } } // Copy Histlist func Copy(hl Histlist) Histlist { - newHl := New() + newHl := New(hl.sugar) // copy list newHl.List = make([]string, len(hl.List)) copy(newHl.List, hl.List) @@ -30,13 +35,16 @@ func Copy(hl Histlist) Histlist { // AddCmdLine to the histlist func (h *Histlist) AddCmdLine(cmdLine string) { - // lenBefore := len(h.List) + // lenBefore := len(h.list) // lookup idx, found := h.LastIndex[cmdLine] if found { // remove duplicate if cmdLine != h.List[idx] { - log.Println("histlist ERROR: Adding cmdLine:", cmdLine, " != LastIndex[cmdLine]:", h.List[idx]) + h.sugar.DPanicw("Index key is different than actual cmd line in the list", + "indexKeyCmdLine", cmdLine, + "actualCmdLine", h.List[idx], + ) } h.List = append(h.List[:idx], h.List[idx+1:]...) // idx++ @@ -44,7 +52,10 @@ func (h *Histlist) AddCmdLine(cmdLine string) { cmdLn := h.List[idx] h.LastIndex[cmdLn]-- if idx != h.LastIndex[cmdLn] { - log.Println("histlist ERROR: Shifting LastIndex idx:", idx, " != LastIndex[cmdLn]:", h.LastIndex[cmdLn]) + h.sugar.DPanicw("Index position is different than actual position of the cmd line", + "actualPosition", idx, + "indexedPosition", h.LastIndex[cmdLn], + ) } idx++ } @@ -53,7 +64,10 @@ func (h *Histlist) AddCmdLine(cmdLine string) { h.LastIndex[cmdLine] = len(h.List) // append new cmdline h.List = append(h.List, cmdLine) - // log.Println("histlist: Added cmdLine:", cmdLine, "; history length:", lenBefore, "->", len(h.List)) + h.sugar.Debugw("Added cmdLine", + "cmdLine", cmdLine, + "historyLength", len(h.List), + ) } // AddHistlist contents of another histlist to this histlist diff --git a/pkg/httpclient/httpclient.go b/internal/httpclient/httpclient.go similarity index 100% rename from pkg/httpclient/httpclient.go rename to internal/httpclient/httpclient.go diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..3167412 --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,27 @@ +package logger + +import ( + "fmt" + "path/filepath" + + "github.com/curusarn/resh/internal/datadir" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func New(executable string, level zapcore.Level, developement bool) (*zap.Logger, error) { + dataDir, err := datadir.GetPath() + if err != nil { + return nil, fmt.Errorf("error while getting resh data dir: %w", err) + } + logPath := filepath.Join(dataDir, "log.json") + loggerConfig := zap.NewProductionConfig() + loggerConfig.OutputPaths = []string{logPath} + loggerConfig.Level.SetLevel(level) + loggerConfig.Development = developement // DPanic panics in developement + logger, err := loggerConfig.Build() + if err != nil { + return logger, fmt.Errorf("error while creating logger: %w", err) + } + return logger.With(zap.String("executable", executable)), err +} diff --git a/internal/msg/msg.go b/internal/msg/msg.go new file mode 100644 index 0000000..7d3b103 --- /dev/null +++ b/internal/msg/msg.go @@ -0,0 +1,21 @@ +package msg + +import "github.com/curusarn/resh/internal/recordint" + +// CliMsg struct +type CliMsg struct { + SessionID string + PWD string +} + +// CliResponse struct +type CliResponse struct { + Records []recordint.SearchApp +} + +// StatusResponse struct +type StatusResponse struct { + Status bool `json:"status"` + Version string `json:"version"` + Commit string `json:"commit"` +} diff --git a/internal/output/output.go b/internal/output/output.go new file mode 100644 index 0000000..e3fd15f --- /dev/null +++ b/internal/output/output.go @@ -0,0 +1,77 @@ +package output + +import ( + "fmt" + "os" + + "go.uber.org/zap" +) + +// Output wrapper for writting to logger and stdout/stderr at the same time +// useful for errors that should be presented to the user +type Output struct { + Logger *zap.Logger + ErrPrefix string +} + +func New(logger *zap.Logger, prefix string) *Output { + return &Output{ + Logger: logger, + ErrPrefix: prefix, + } +} + +func (f *Output) Info(msg string) { + fmt.Fprintf(os.Stdout, msg) + f.Logger.Info(msg) +} + +func (f *Output) Error(msg string, err error) { + fmt.Fprintf(os.Stderr, "%s: %s: %v", f.ErrPrefix, msg, err) + f.Logger.Error(msg, zap.Error(err)) +} + +func (f *Output) Fatal(msg string, err error) { + fmt.Fprintf(os.Stderr, "%s: %s: %v", f.ErrPrefix, msg, err) + f.Logger.Fatal(msg, zap.Error(err)) +} + +var msgDeamonNotRunning = `Resh-daemon didn't respond - it's probably not running. + + -> Try restarting this terminal window to bring resh-daemon back up + -> If the problem persists you can check resh-daemon logs: ~/.local/share/resh/log.json (or ~/$XDG_DATA_HOME/resh/log.json) + -> You can create an issue at: https://github.com/curusarn/resh/issues + +` +var msgVersionMismatch = `This terminal session was started with different resh version than is installed now. +It looks like you updated resh and didn't restart this terminal. + + -> Restart this terminal window to fix that + +` + +func (f *Output) ErrorDaemonNotRunning(err error) { + fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgDeamonNotRunning) + f.Logger.Error("Daemon is not running", zap.Error(err)) +} + +func (f *Output) FatalDaemonNotRunning(err error) { + fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgDeamonNotRunning) + f.Logger.Fatal("Daemon is not running", zap.Error(err)) +} + +func (f *Output) ErrorVersionMismatch(installedVer, terminalVer string) { + fmt.Fprintf(os.Stderr, "%s: %s\n\n(installed version: %s, this terminal version: %s)", + f.ErrPrefix, msgVersionMismatch, installedVer, terminalVer) + f.Logger.Fatal("Version mismatch", + zap.String("installed", installedVer), + zap.String("terminal", terminalVer)) +} + +func (f *Output) FatalVersionMismatch(installedVer, terminalVer string) { + fmt.Fprintf(os.Stderr, "%s: %s\n(installed version: %s, this terminal version: %s)\n", + f.ErrPrefix, msgVersionMismatch, installedVer, terminalVer) + f.Logger.Fatal("Version mismatch", + zap.String("installed", installedVer), + zap.String("terminal", terminalVer)) +} diff --git a/internal/recconv/recconv.go b/internal/recconv/recconv.go new file mode 100644 index 0000000..7690ec4 --- /dev/null +++ b/internal/recconv/recconv.go @@ -0,0 +1,37 @@ +package recconv + +import ( + "fmt" + + "github.com/curusarn/resh/record" +) + +func LegacyToV1(r *record.Legacy) *record.V1 { + return &record.V1{ + // FIXME: fill in all the fields + + // Flags: 0, + + CmdLine: r.CmdLine, + ExitCode: r.ExitCode, + + DeviceID: r.ReshUUID, + SessionID: r.SessionID, + RecordID: r.RecordID, + + Home: r.Home, + Pwd: r.Pwd, + RealPwd: r.RealPwd, + + // Logname: r.Login, + Device: r.Host, + + GitOriginRemote: r.GitOriginRemote, + + Time: fmt.Sprintf("%.4f", r.RealtimeBefore), + Duration: fmt.Sprintf("%.4f", r.RealtimeDuration), + + PartOne: r.PartOne, + PartsNotMerged: !r.PartsMerged, + } +} diff --git a/internal/recio/read.go b/internal/recio/read.go new file mode 100644 index 0000000..b71cbb6 --- /dev/null +++ b/internal/recio/read.go @@ -0,0 +1,158 @@ +package recio + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/curusarn/resh/internal/recconv" + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" + "go.uber.org/zap" +) + +func (r *RecIO) ReadAndFixFile(fpath string, maxErrors int) ([]recordint.Indexed, error) { + recs, numErrs, err := r.ReadFile(fpath) + if err != nil { + return nil, err + } + if numErrs > maxErrors { + return nil, fmt.Errorf("encountered too many decoding errors") + } + if numErrs == 0 { + return recs, nil + } + + // TODO: check there error messages + r.sugar.Warnw("Some history records could not be decoded - fixing resh history file by dropping them", + "corruptedRecords", numErrs, + ) + fpathBak := fpath + ".bak" + r.sugar.Infow("Backing up current corrupted history file", + "backupFilename", fpathBak, + ) + // TODO: maybe use upstram copy function + err = copyFile(fpath, fpathBak) + if err != nil { + r.sugar.Errorw("Failed to create a backup history file - aborting fixing history file", + "backupFilename", fpathBak, + zap.Error(err), + ) + return recs, nil + } + r.sugar.Info("Writing resh history file without errors ...") + var recsV1 []record.V1 + for _, rec := range recs { + recsV1 = append(recsV1, rec.Rec) + } + err = r.OverwriteFile(fpath, recsV1) + if err != nil { + r.sugar.Errorw("Failed write fixed history file - aborting fixing history file", + "filename", fpath, + zap.Error(err), + ) + } + return recs, nil +} + +func (r *RecIO) ReadFile(fpath string) ([]recordint.Indexed, int, error) { + var recs []recordint.Indexed + file, err := os.Open(fpath) + if err != nil { + return nil, 0, fmt.Errorf("failed to open history file: %w", err) + } + defer file.Close() + + reader := bufio.NewReader(file) + numErrs := 0 + var idx int + for { + var line string + line, err = reader.ReadString('\n') + if err != nil { + break + } + idx++ + rec, err := r.decodeLine(line) + if err != nil { + numErrs++ + continue + } + recidx := recordint.Indexed{ + Rec: *rec, + // TODO: Is line index actually enough? + // Don't we want to count bytes because we will scan by number of bytes? + // hint: https://benjamincongdon.me/blog/2018/04/10/Counting-Scanned-Bytes-in-Go/ + Idx: idx, + } + recs = append(recs, recidx) + } + if err != io.EOF { + r.sugar.Error("Error while loading file", zap.Error(err)) + } + r.sugar.Infow("Loaded resh history records", + "recordCount", len(recs), + ) + return recs, numErrs, nil +} + +func copyFile(source, dest string) error { + from, err := os.Open(source) + if err != nil { + return err + } + defer from.Close() + + // This is equivalnet to: os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666) + to, err := os.Create(dest) + if err != nil { + return err + } + defer to.Close() + + _, err = io.Copy(to, from) + if err != nil { + return err + } + return nil +} + +func (r *RecIO) decodeLine(line string) (*record.V1, error) { + idx := strings.Index(line, "{") + if idx == -1 { + return nil, fmt.Errorf("no openning brace found") + } + schema := line[:idx] + jsn := line[idx:] + switch schema { + case "v1": + var rec record.V1 + err := decodeAnyRecord(jsn, &rec) + if err != nil { + return nil, err + } + return &rec, nil + case "": + var rec record.Legacy + err := decodeAnyRecord(jsn, &rec) + if err != nil { + return nil, err + } + return recconv.LegacyToV1(&rec), nil + default: + return nil, fmt.Errorf("unknown record schema/type '%s'", schema) + } +} + +// TODO: find out if we are loosing performance because of the use of interface{} + +func decodeAnyRecord(jsn string, rec interface{}) error { + err := json.Unmarshal([]byte(jsn), &rec) + if err != nil { + return fmt.Errorf("failed to decode json: %w", err) + } + return nil +} diff --git a/internal/recio/recio.go b/internal/recio/recio.go new file mode 100644 index 0000000..5ea986b --- /dev/null +++ b/internal/recio/recio.go @@ -0,0 +1,13 @@ +package recio + +import ( + "go.uber.org/zap" +) + +type RecIO struct { + sugar *zap.SugaredLogger +} + +func New(sugar *zap.SugaredLogger) RecIO { + return RecIO{sugar: sugar} +} diff --git a/internal/recio/write.go b/internal/recio/write.go new file mode 100644 index 0000000..1ff4506 --- /dev/null +++ b/internal/recio/write.go @@ -0,0 +1,62 @@ +package recio + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" +) + +// TODO: better errors +func (r *RecIO) OverwriteFile(fpath string, recs []record.V1) error { + file, err := os.Create(fpath) + if err != nil { + return err + } + defer file.Close() + return writeRecords(file, recs) +} + +// TODO: better errors +func (r *RecIO) AppendToFile(fpath string, recs []record.V1) error { + file, err := os.OpenFile(fpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + return writeRecords(file, recs) +} + +// TODO: better errors +func (r *RecIO) EditRecordFlagsInFile(fpath string, idx int, rec recordint.Flag) error { + // FIXME: implement + // open file "not as append" + // scan to the correct line + r.sugar.Error("not implemented yet (FIXME)") + return nil +} + +func writeRecords(file *os.File, recs []record.V1) error { + for _, rec := range recs { + jsn, err := encodeV1Record(rec) + if err != nil { + return err + } + _, err = file.Write(jsn) + if err != nil { + return err + } + } + return nil +} + +func encodeV1Record(rec record.V1) ([]byte, error) { + version := []byte("v1") + jsn, err := json.Marshal(rec) + if err != nil { + return nil, fmt.Errorf("failed to encode json: %w", err) + } + return append(append(version, jsn...), []byte("\n")...), nil +} diff --git a/internal/recordint/collect.go b/internal/recordint/collect.go new file mode 100644 index 0000000..1a7d6a7 --- /dev/null +++ b/internal/recordint/collect.go @@ -0,0 +1,34 @@ +package recordint + +import "github.com/curusarn/resh/record" + +type Collect struct { + // record merging + SessionID string + Shlvl int + // session watching + SessionPID int + Shell string + + Rec record.V1 +} + +type Postcollect struct { + // record merging + SessionID string + Shlvl int + // session watching + SessionPID int + + RecordID string + ExitCode int + Duration float64 +} + +type SessionInit struct { + // record merging + SessionID string + Shlvl int + // session watching + SessionPID int +} diff --git a/internal/recordint/flag.go b/internal/recordint/flag.go new file mode 100644 index 0000000..2eaff6a --- /dev/null +++ b/internal/recordint/flag.go @@ -0,0 +1,9 @@ +package recordint + +type Flag struct { + deviceID string + recordID string + + flagDeleted bool + flagFavourite bool +} diff --git a/internal/recordint/indexed.go b/internal/recordint/indexed.go new file mode 100644 index 0000000..950cadf --- /dev/null +++ b/internal/recordint/indexed.go @@ -0,0 +1,9 @@ +package recordint + +import "github.com/curusarn/resh/record" + +// Indexed record allows us to find records in history file in order to edit them +type Indexed struct { + Rec record.V1 + Idx int +} diff --git a/internal/recordint/recordint.go b/internal/recordint/recordint.go new file mode 100644 index 0000000..73457cd --- /dev/null +++ b/internal/recordint/recordint.go @@ -0,0 +1,2 @@ +// Package recordint provides internal record types that are passed between resh components +package recordint diff --git a/internal/recordint/searchapp.go b/internal/recordint/searchapp.go new file mode 100644 index 0000000..1147877 --- /dev/null +++ b/internal/recordint/searchapp.go @@ -0,0 +1,77 @@ +package recordint + +import ( + "net/url" + "strconv" + "strings" + + giturls "github.com/whilp/git-urls" +) + +// SearchApp record used for sending records to RESH-CLI +type SearchApp struct { + IsRaw bool + SessionID string + DeviceID string + + CmdLine string + Host string + Pwd string + Home string // helps us to collapse /home/user to tilde + GitOriginRemote string + ExitCode int + + Time float64 + + // file index + Idx int +} + +// NewCliRecordFromCmdLine +func NewSearchAppFromCmdLine(cmdLine string) SearchApp { + return SearchApp{ + IsRaw: true, + CmdLine: cmdLine, + } +} + +// NewCliRecord from EnrichedRecord +func NewSearchApp(r *Indexed) SearchApp { + // TODO: we used to validate records with recutil.Validate() + // TODO: handle this error + time, _ := strconv.ParseFloat(r.Rec.Time, 64) + return SearchApp{ + IsRaw: false, + SessionID: r.Rec.SessionID, + DeviceID: r.Rec.DeviceID, + CmdLine: r.Rec.CmdLine, + Host: r.Rec.Device, + Pwd: r.Rec.Pwd, + Home: r.Rec.Home, + // TODO: is this the right place to normalize the git remote + GitOriginRemote: normalizeGitRemote(r.Rec.GitOriginRemote), + ExitCode: r.Rec.ExitCode, + Time: time, + + Idx: r.Idx, + } +} + +// TODO: maybe move this to a more appropriate place +// normalizeGitRemote helper +func normalizeGitRemote(gitRemote string) string { + if strings.HasSuffix(gitRemote, ".git") { + gitRemote = gitRemote[:len(gitRemote)-4] + } + parsedURL, err := giturls.Parse(gitRemote) + if err != nil { + // TODO: log this error + return gitRemote + } + if parsedURL.User == nil || parsedURL.User.Username() == "" { + parsedURL.User = url.User("git") + } + // TODO: figure out what scheme we want + parsedURL.Scheme = "git+ssh" + return parsedURL.String() +} diff --git a/internal/records/records.go b/internal/records/records.go new file mode 100644 index 0000000..a9972d2 --- /dev/null +++ b/internal/records/records.go @@ -0,0 +1,79 @@ +package records + +import ( + "bufio" + "os" + "strings" + + "github.com/curusarn/resh/internal/histlist" + "go.uber.org/zap" +) + +// LoadCmdLinesFromZshFile loads cmdlines from zsh history file +func LoadCmdLinesFromZshFile(sugar *zap.SugaredLogger, fname string) histlist.Histlist { + hl := histlist.New(sugar) + file, err := os.Open(fname) + if err != nil { + sugar.Error("Failed to open zsh history file - skipping reading zsh history", zap.Error(err)) + return hl + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + // trim newline + line = strings.TrimRight(line, "\n") + var cmd string + // zsh format EXTENDED_HISTORY + // : 1576270617:0;make install + // zsh format no EXTENDED_HISTORY + // make install + if len(line) == 0 { + // skip empty + continue + } + if strings.Contains(line, ":") && strings.Contains(line, ";") && + len(strings.Split(line, ":")) >= 3 && len(strings.Split(line, ";")) >= 2 { + // contains at least 2x ':' and 1x ';' => assume EXTENDED_HISTORY + cmd = strings.Split(line, ";")[1] + } else { + cmd = line + } + hl.AddCmdLine(cmd) + } + return hl +} + +// LoadCmdLinesFromBashFile loads cmdlines from bash history file +func LoadCmdLinesFromBashFile(sugar *zap.SugaredLogger, fname string) histlist.Histlist { + hl := histlist.New(sugar) + file, err := os.Open(fname) + if err != nil { + sugar.Error("Failed to open bash history file - skipping reading bash history", zap.Error(err)) + return hl + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + // trim newline + line = strings.TrimRight(line, "\n") + // trim spaces from left + line = strings.TrimLeft(line, " ") + // bash format (two lines) + // #1576199174 + // make install + if strings.HasPrefix(line, "#") { + // is either timestamp or comment => skip + continue + } + if len(line) == 0 { + // skip empty + continue + } + hl.AddCmdLine(line) + } + return hl +} diff --git a/internal/recutil/recutil.go b/internal/recutil/recutil.go new file mode 100644 index 0000000..53b1cff --- /dev/null +++ b/internal/recutil/recutil.go @@ -0,0 +1,51 @@ +package recutil + +import ( + "errors" + + "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" +) + +// TODO: reintroduce validation +// Validate returns error if the record is invalid +// func Validate(r *record.V1) error { +// if r.CmdLine == "" { +// return errors.New("There is no CmdLine") +// } +// if r.Time == 0 { +// return errors.New("There is no Time") +// } +// if r.RealPwd == "" { +// return errors.New("There is no Real Pwd") +// } +// if r.Pwd == "" { +// return errors.New("There is no Pwd") +// } +// return nil +// } + +// TODO: maybe more to a more appropriate place +// TODO: cleanup the interface - stop modifying the part1 and returning a ew record at the same time +// Merge two records (part1 - collect + part2 - postcollect) +func Merge(r1 *recordint.Collect, r2 *recordint.Collect) (record.V1, error) { + if r1.SessionID != r2.SessionID { + return record.V1{}, errors.New("Records to merge are not from the same sesion - r1:" + r1.SessionID + " r2:" + r2.SessionID) + } + if r1.Rec.RecordID != r2.Rec.RecordID { + return record.V1{}, errors.New("Records to merge do not have the same ID - r1:" + r1.Rec.RecordID + " r2:" + r2.Rec.RecordID) + } + + r := recordint.Collect{ + SessionID: r1.SessionID, + Shlvl: r1.Shlvl, + SessionPID: r1.SessionPID, + + Rec: r1.Rec, + } + r.Rec.ExitCode = r2.Rec.ExitCode + r.Rec.Duration = r2.Rec.Duration + r.Rec.PartOne = false + r.Rec.PartsNotMerged = false + return r.Rec, nil +} diff --git a/pkg/searchapp/highlight.go b/internal/searchapp/highlight.go similarity index 100% rename from pkg/searchapp/highlight.go rename to internal/searchapp/highlight.go diff --git a/pkg/searchapp/item.go b/internal/searchapp/item.go similarity index 94% rename from pkg/searchapp/item.go rename to internal/searchapp/item.go index 6908c25..33fc827 100644 --- a/pkg/searchapp/item.go +++ b/internal/searchapp/item.go @@ -2,13 +2,12 @@ package searchapp import ( "fmt" - "log" "math" "strconv" "strings" "time" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/recordint" "golang.org/x/exp/utf8string" ) @@ -19,7 +18,7 @@ const dots = "…" type Item struct { isRaw bool - realtimeBefore float64 + time float64 // [host:]pwd differentHost bool @@ -106,8 +105,8 @@ func (i Item) DrawStatusLine(compactRendering bool, printedLineLength, realLineL if i.isRaw { return splitStatusLineToLines(i.CmdLine, printedLineLength, realLineLength) } - secs := int64(i.realtimeBefore) - nsecs := int64((i.realtimeBefore - float64(secs)) * 1e9) + secs := int64(i.time) + nsecs := int64((i.time - float64(secs)) * 1e9) tm := time.Unix(secs, nsecs) const timeFormat = "2006-01-02 15:04:05" timeString := tm.Format(timeFormat) @@ -143,8 +142,8 @@ func (i Item) DrawItemColumns(compactRendering bool, debug bool) ItemColumns { // DISPLAY // DISPLAY > date - secs := int64(i.realtimeBefore) - nsecs := int64((i.realtimeBefore - float64(secs)) * 1e9) + secs := int64(i.time) + nsecs := int64((i.time - float64(secs)) * 1e9) tm := time.Unix(secs, nsecs) var date string @@ -228,9 +227,6 @@ func produceLocation(length int, host string, pwdTilde string, differentHost boo shrinkFactor := float64(length) / float64(totalLen) shrinkedHostLen := int(math.Ceil(float64(hostLen) * shrinkFactor)) - if debug { - log.Printf("shrinkFactor: %f\n", shrinkFactor) - } halfLocationLen := length/2 - colonLen newHostLen = minInt(hostLen, shrinkedHostLen, halfLocationLen) @@ -318,7 +314,7 @@ func properMatch(str, term, padChar string) bool { // NewItemFromRecordForQuery creates new item from record based on given query // returns error if the query doesn't match the record -func NewItemFromRecordForQuery(record records.CliRecord, query Query, debug bool) (Item, error) { +func NewItemFromRecordForQuery(record recordint.SearchApp, query Query, debug bool) (Item, error) { // Use numbers that won't add up to same score for any number of query words // query score weigth 1.51 const hitScore = 1.517 // 1 * 1.51 @@ -415,10 +411,10 @@ func NewItemFromRecordForQuery(record records.CliRecord, query Query, debug bool // if score <= 0 && !anyHit { // return Item{}, errors.New("no match for given record and query") // } - score += record.RealtimeBefore * timeScoreCoef + score += record.Time * timeScoreCoef it := Item{ - realtimeBefore: record.RealtimeBefore, + time: record.Time, differentHost: differentHost, host: record.Host, @@ -474,7 +470,7 @@ type RawItem struct { // NewRawItemFromRecordForQuery creates new item from record based on given query // returns error if the query doesn't match the record -func NewRawItemFromRecordForQuery(record records.CliRecord, terms []string, debug bool) (RawItem, error) { +func NewRawItemFromRecordForQuery(record recordint.SearchApp, terms []string, debug bool) (RawItem, error) { const hitScore = 1.0 const hitScoreConsecutive = 0.01 const properMatchScore = 0.3 @@ -493,7 +489,7 @@ func NewRawItemFromRecordForQuery(record records.CliRecord, terms []string, debu cmd = strings.ReplaceAll(cmd, term, highlightMatch(term)) } } - score += record.RealtimeBefore * timeScoreCoef + score += record.Time * timeScoreCoef // KEY for deduplication key := record.CmdLine diff --git a/pkg/searchapp/item_test.go b/internal/searchapp/item_test.go similarity index 100% rename from pkg/searchapp/item_test.go rename to internal/searchapp/item_test.go diff --git a/pkg/searchapp/query.go b/internal/searchapp/query.go similarity index 80% rename from pkg/searchapp/query.go rename to internal/searchapp/query.go index 2b8227d..fc2870a 100644 --- a/pkg/searchapp/query.go +++ b/internal/searchapp/query.go @@ -1,7 +1,6 @@ package searchapp import ( - "log" "sort" "strings" ) @@ -37,26 +36,16 @@ func filterTerms(terms []string) []string { // NewQueryFromString . func NewQueryFromString(queryInput string, host string, pwd string, gitOriginRemote string, debug bool) Query { - if debug { - log.Println("QUERY input = <" + queryInput + ">") - } terms := strings.Fields(queryInput) var logStr string for _, term := range terms { logStr += " <" + term + ">" } - if debug { - log.Println("QUERY raw terms =" + logStr) - } terms = filterTerms(terms) logStr = "" for _, term := range terms { logStr += " <" + term + ">" } - if debug { - log.Println("QUERY filtered terms =" + logStr) - log.Println("QUERY pwd =" + pwd) - } sort.SliceStable(terms, func(i, j int) bool { return len(terms[i]) < len(terms[j]) }) return Query{ terms: terms, @@ -68,17 +57,11 @@ func NewQueryFromString(queryInput string, host string, pwd string, gitOriginRem // GetRawTermsFromString . func GetRawTermsFromString(queryInput string, debug bool) []string { - if debug { - log.Println("QUERY input = <" + queryInput + ">") - } terms := strings.Fields(queryInput) var logStr string for _, term := range terms { logStr += " <" + term + ">" } - if debug { - log.Println("QUERY raw terms =" + logStr) - } terms = filterTerms(terms) logStr = "" for _, term := range terms { diff --git a/internal/searchapp/test.go b/internal/searchapp/test.go new file mode 100644 index 0000000..d89b805 --- /dev/null +++ b/internal/searchapp/test.go @@ -0,0 +1,26 @@ +package searchapp + +import ( + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/msg" + "github.com/curusarn/resh/internal/recio" + "go.uber.org/zap" +) + +// LoadHistoryFromFile ... +func LoadHistoryFromFile(sugar *zap.SugaredLogger, historyPath string, numLines int) msg.CliResponse { + rio := recio.New(sugar) + recs, _, err := rio.ReadFile(historyPath) + if err != nil { + sugar.Panicf("failed to read hisotry file: %w", err) + } + if numLines != 0 && numLines < len(recs) { + recs = recs[:numLines] + } + cliRecords := histcli.New(sugar) + for i := len(recs) - 1; i >= 0; i-- { + rec := recs[i] + cliRecords.AddRecord(&rec) + } + return msg.CliResponse{Records: cliRecords.Dump()} +} diff --git a/pkg/searchapp/time.go b/internal/searchapp/time.go similarity index 100% rename from pkg/searchapp/time.go rename to internal/searchapp/time.go diff --git a/pkg/sess/sess.go b/internal/sess/sess.go similarity index 100% rename from pkg/sess/sess.go rename to internal/sess/sess.go diff --git a/internal/sesswatch/sesswatch.go b/internal/sesswatch/sesswatch.go new file mode 100644 index 0000000..b20bb61 --- /dev/null +++ b/internal/sesswatch/sesswatch.go @@ -0,0 +1,96 @@ +package sesswatch + +import ( + "sync" + "time" + + "github.com/curusarn/resh/internal/recordint" + "github.com/mitchellh/go-ps" + "go.uber.org/zap" +) + +type sesswatch struct { + sugar *zap.SugaredLogger + + sessionsToDrop []chan string + sleepSeconds uint + + watchedSessions map[string]bool + mutex sync.Mutex +} + +// Go runs the session watcher - watches sessions and sends +func Go(sugar *zap.SugaredLogger, + sessionsToWatch chan recordint.SessionInit, sessionsToWatchRecords chan recordint.Collect, + sessionsToDrop []chan string, sleepSeconds uint) { + + sw := sesswatch{ + sugar: sugar.With("module", "sesswatch"), + sessionsToDrop: sessionsToDrop, + sleepSeconds: sleepSeconds, + watchedSessions: map[string]bool{}, + } + go sw.waiter(sessionsToWatch, sessionsToWatchRecords) +} + +func (s *sesswatch) waiter(sessionsToWatch chan recordint.SessionInit, sessionsToWatchRecords chan recordint.Collect) { + for { + func() { + select { + case rec := <-sessionsToWatch: + // normal way to start watching a session + id := rec.SessionID + pid := rec.SessionPID + sugar := s.sugar.With( + "sessionID", rec.SessionID, + "sessionPID", rec.SessionPID, + ) + s.mutex.Lock() + defer s.mutex.Unlock() + if s.watchedSessions[id] == false { + sugar.Infow("Starting watching new session") + s.watchedSessions[id] = true + go s.watcher(sugar, id, pid) + } + case rec := <-sessionsToWatchRecords: + // additional safety - watch sessions that were never properly initialized + id := rec.SessionID + pid := rec.SessionPID + sugar := s.sugar.With( + "sessionID", rec.SessionID, + "sessionPID", rec.SessionPID, + ) + s.mutex.Lock() + defer s.mutex.Unlock() + if s.watchedSessions[id] == false { + sugar.Warnw("Starting watching new session based on '/record'") + s.watchedSessions[id] = true + go s.watcher(sugar, id, pid) + } + } + }() + } +} + +func (s *sesswatch) watcher(sugar *zap.SugaredLogger, sessionID string, sessionPID int) { + for { + time.Sleep(time.Duration(s.sleepSeconds) * time.Second) + proc, err := ps.FindProcess(sessionPID) + if err != nil { + sugar.Errorw("Error while finding process", "error", err) + } else if proc == nil { + sugar.Infow("Dropping session") + func() { + s.mutex.Lock() + defer s.mutex.Unlock() + s.watchedSessions[sessionID] = false + }() + for _, ch := range s.sessionsToDrop { + sugar.Debugw("Sending 'drop session' message ...") + ch <- sessionID + sugar.Debugw("Sending 'drop session' message DONE") + } + break + } + } +} diff --git a/internal/signalhandler/signalhander.go b/internal/signalhandler/signalhander.go new file mode 100644 index 0000000..b8188d6 --- /dev/null +++ b/internal/signalhandler/signalhander.go @@ -0,0 +1,74 @@ +package signalhandler + +import ( + "context" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "go.uber.org/zap" +) + +func sendSignals(sugar *zap.SugaredLogger, sig os.Signal, subscribers []chan os.Signal, done chan string) { + for _, sub := range subscribers { + sub <- sig + } + sugar.Warnw("Sent shutdown signals to components") + chanCount := len(subscribers) + start := time.Now() + delay := time.Millisecond * 100 + timeout := time.Millisecond * 2000 + + for { + select { + case _ = <-done: + chanCount-- + if chanCount == 0 { + sugar.Warnw("All components shut down successfully") + return + } + default: + time.Sleep(delay) + } + if time.Since(start) > timeout { + sugar.Errorw("Timouted while waiting for proper shutdown", + "componentsStillUp", strconv.Itoa(chanCount), + "timeout", timeout.String(), + ) + return + } + } +} + +// Run catches and handles signals +func Run(sugar *zap.SugaredLogger, subscribers []chan os.Signal, done chan string, server *http.Server) { + sugar = sugar.With("module", "signalhandler") + signals := make(chan os.Signal, 1) + + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + var sig os.Signal + for { + sig := <-signals + sugarSig := sugar.With("signal", sig.String()) + sugarSig.Infow("Got signal") + if sig == syscall.SIGTERM { + // Shutdown daemon on SIGTERM + break + } + sugarSig.Warnw("Ignoring signal. Send SIGTERM to trigger shutdown.") + } + + sugar.Infow("Sending shutdown signals to components ...") + sendSignals(sugar, sig, subscribers, done) + + sugar.Infow("Shutting down the server ...") + if err := server.Shutdown(context.Background()); err != nil { + sugar.Errorw("Error while shuting down HTTP server", + "error", err, + ) + } +} diff --git a/internal/syncconnector/reader.go b/internal/syncconnector/reader.go new file mode 100644 index 0000000..729a6d7 --- /dev/null +++ b/internal/syncconnector/reader.go @@ -0,0 +1,118 @@ +package syncconnector + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/curusarn/resh/record" + "io" + "net/http" + "strconv" + "time" +) + +func (sc SyncConnector) getLatestRecord(machineId *string) (map[string]string, error) { + return map[string]string{}, nil +} + +func (sc SyncConnector) downloadRecords(lastRecords map[string]float64) ([]record.V1, error) { + var records []record.V1 + + client := http.Client{ + Timeout: 3 * time.Second, + } + + latestRes := map[string]string{} + for device, t := range lastRecords { + sc.sugar.Debugf("Latest for %s is %f", device, t) + latestRes[device] = fmt.Sprintf("%.4f", t) + } + + latestJson, err := json.Marshal(latestRes) + if err != nil { + sc.sugar.Errorw("converting latest to JSON failed", "err", err) + return nil, err + } + reqBody := bytes.NewBuffer(latestJson) + + address := sc.getAddressWithPath(historyEndpoint) + resp, err := client.Post(address, "application/json", reqBody) + if err != nil { + sc.sugar.Errorw("history request failed", "address", address, "err", err) + return nil, err + } + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + sc.sugar.Errorw("reader close failed", "err", err) + } + }(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + sc.sugar.Warnw("reading response body failed", "err", err) + } + + err = json.Unmarshal(body, &records) + if err != nil { + sc.sugar.Errorw("Unmarshalling failed", "err", err) + return nil, err + } + + return records, nil +} + +func (sc SyncConnector) latest() (map[string]float64, error) { + var knownDevices []string + for deviceId, _ := range sc.history.LatestRecordsPerDevice() { + knownDevices = append(knownDevices, deviceId) + } + + client := http.Client{ + Timeout: 3 * time.Second, + } + + knownJson, err := json.Marshal(knownDevices) + if err != nil { + sc.sugar.Errorw("converting latest to JSON failed", "err", err) + return nil, err + } + reqBody := bytes.NewBuffer(knownJson) + + address := sc.getAddressWithPath(latestEndpoint) + resp, err := client.Post(address, "application/json", reqBody) + if err != nil { + sc.sugar.Errorw("latest request failed", "address", address, "err", err) + return nil, err + } + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + sc.sugar.Errorw("reader close failed", "err", err) + } + }(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + sc.sugar.Warnw("reading response body failed", "err", err) + } + + latest := map[string]string{} + + err = json.Unmarshal(body, &latest) + if err != nil { + sc.sugar.Errorw("Unmarshalling failed", "err", err) + return nil, err + } + + l := make(map[string]float64, len(latest)) + for deviceId, ts := range latest { + t, err := strconv.ParseFloat(ts, 64) + if err != nil { + return nil, err + } + l[deviceId] = t + } + + return l, nil +} diff --git a/internal/syncconnector/syncconnector.go b/internal/syncconnector/syncconnector.go new file mode 100644 index 0000000..f1df3e0 --- /dev/null +++ b/internal/syncconnector/syncconnector.go @@ -0,0 +1,79 @@ +package syncconnector + +import ( + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" + "net/url" + "path" + "time" +) + +const storeEndpoint = "/store" +const historyEndpoint = "/history" +const latestEndpoint = "/latest" + +type SyncConnector struct { + sugar *zap.SugaredLogger + + address *url.URL + authToken string + + history *histcli.Histcli +} + +func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodSeconds int, sendPeriodSeconds int, history *histcli.Histcli) (*SyncConnector, error) { + parsedAddress, err := url.Parse(address) + if err != nil { + return nil, err + } + + sc := &SyncConnector{ + sugar: sugar.With(zap.String("component", "syncConnector")), + authToken: authToken, + address: parsedAddress, + history: history, + } + + // TODO: propagate signals + go func(sc *SyncConnector) { + for _ = range time.Tick(time.Second * time.Duration(pullPeriodSeconds)) { + sc.sugar.Debug("checking remote for new records") + + recs, err := sc.downloadRecords(sc.history.LatestRecordsPerDevice()) + if err != nil { + continue + } + + sc.sugar.Debugf("Got %d records", len(recs)) + + for _, rec := range recs { + sc.history.AddRecord(&recordint.Indexed{ + Rec: rec, + }) + } + + } + }(sc) + + go func(sc *SyncConnector) { + // wait to properly load all the records + time.Sleep(time.Second * time.Duration(sendPeriodSeconds)) + for _ = range time.Tick(time.Second * time.Duration(sendPeriodSeconds)) { + sc.sugar.Debug("syncing local records to the remote") + + err := sc.write() + if err != nil { + sc.sugar.Warnw("sending records to the remote failed", "err", err) + } + } + }(sc) + + return sc, nil +} + +func (sc SyncConnector) getAddressWithPath(endpoint string) string { + address := *sc.address + address.Path = path.Join(address.Path, endpoint) + return address.String() +} diff --git a/internal/syncconnector/writer.go b/internal/syncconnector/writer.go new file mode 100644 index 0000000..cd935cc --- /dev/null +++ b/internal/syncconnector/writer.go @@ -0,0 +1,77 @@ +package syncconnector + +import ( + "bytes" + "encoding/json" + "github.com/curusarn/resh/record" + "io" + "net/http" + "strconv" + "time" +) + +func (sc SyncConnector) write() error { + latestRemote, err := sc.latest() + if err != nil { + return err + } + latestLocal := sc.history.LatestRecordsPerDevice() + remoteIsOlder := false + for deviceId, lastLocal := range latestLocal { + if lastRemote, ok := latestRemote[deviceId]; !ok { + // Unknown deviceId on the remote - add records have to be sent + remoteIsOlder = true + break + } else if lastLocal > lastRemote { + remoteIsOlder = true + break + } + } + if !remoteIsOlder { + sc.sugar.Debug("No need to sync remote, there are no newer local records") + return nil + } + var toSend []record.V1 + for _, r := range sc.history.DumpRaw() { + t, err := strconv.ParseFloat(r.Time, 64) + if err != nil { + sc.sugar.Warnw("Invalid time for record - skipping", "time", r.Time) + continue + } + l, ok := latestRemote[r.DeviceID] + if ok && l >= t { + continue + } + sc.sugar.Infow("record is newer", "new", t, "old", l, "id", r.RecordID, "deviceid", r.DeviceID) + toSend = append(toSend, r) + } + + client := http.Client{ + Timeout: 3 * time.Second, + } + + toSendJson, err := json.Marshal(toSend) + if err != nil { + sc.sugar.Errorw("converting toSend to JSON failed", "err", err) + return err + } + reqBody := bytes.NewBuffer(toSendJson) + + address := sc.getAddressWithPath(storeEndpoint) + resp, err := client.Post(address, "application/json", reqBody) + if err != nil { + sc.sugar.Errorw("store request failed", "address", address, "err", err) + return err + } + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + sc.sugar.Errorw("reader close failed", "err", err) + } + }(resp.Body) + + sc.sugar.Debugw("store call", "status", resp.Status) + + return nil +} diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go deleted file mode 100644 index 06cc44d..0000000 --- a/pkg/cfg/cfg.go +++ /dev/null @@ -1,12 +0,0 @@ -package cfg - -// Config struct -type Config struct { - Port int - SesswatchPeriodSeconds uint - SesshistInitHistorySize int - Debug bool - BindArrowKeysBash bool - BindArrowKeysZsh bool - BindControlR bool -} diff --git a/pkg/collect/collect.go b/pkg/collect/collect.go deleted file mode 100644 index 5fd849b..0000000 --- a/pkg/collect/collect.go +++ /dev/null @@ -1,120 +0,0 @@ -package collect - -import ( - "bytes" - "encoding/json" - "io/ioutil" - "log" - "net/http" - "path/filepath" - "strconv" - "strings" - - "github.com/curusarn/resh/pkg/httpclient" - "github.com/curusarn/resh/pkg/records" -) - -// SingleResponse json struct -type SingleResponse struct { - Found bool `json:"found"` - CmdLine string `json:"cmdline"` -} - -// SendRecallRequest to daemon -func SendRecallRequest(r records.SlimRecord, port string) (string, bool) { - recJSON, err := json.Marshal(r) - if err != nil { - log.Fatal("send err 1", err) - } - - req, err := http.NewRequest("POST", "http://localhost:"+port+"/recall", - bytes.NewBuffer(recJSON)) - if err != nil { - log.Fatal("send err 2", err) - } - req.Header.Set("Content-Type", "application/json") - - client := httpclient.New() - resp, err := client.Do(req) - if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal("read response error") - } - log.Println(string(body)) - response := SingleResponse{} - err = json.Unmarshal(body, &response) - if err != nil { - log.Fatal("unmarshal resp error: ", err) - } - log.Println(response) - return response.CmdLine, response.Found -} - -// SendRecord to daemon -func SendRecord(r records.Record, port, path string) { - recJSON, err := json.Marshal(r) - if err != nil { - log.Fatal("send err 1", err) - } - - req, err := http.NewRequest("POST", "http://localhost:"+port+path, - bytes.NewBuffer(recJSON)) - if err != nil { - log.Fatal("send err 2", err) - } - req.Header.Set("Content-Type", "application/json") - - client := httpclient.New() - _, err = client.Do(req) - if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") - } -} - -// ReadFileContent and return it as a string -func ReadFileContent(path string) string { - dat, err := ioutil.ReadFile(path) - if err != nil { - return "" - //log.Fatal("failed to open " + path) - } - return strings.TrimSuffix(string(dat), "\n") -} - -// GetGitDirs based on result of git "cdup" command -func GetGitDirs(cdup string, exitCode int, pwd string) (string, string) { - if exitCode != 0 { - return "", "" - } - abspath := filepath.Clean(filepath.Join(pwd, cdup)) - realpath, err := filepath.EvalSymlinks(abspath) - if err != nil { - log.Println("err while handling git dir paths:", err) - return "", "" - } - return abspath, realpath -} - -// GetTimezoneOffsetInSeconds based on zone returned by date command -func GetTimezoneOffsetInSeconds(zone string) float64 { - // date +%z -> "+0200" - hoursStr := zone[:3] - minsStr := zone[3:] - hours, err := strconv.Atoi(hoursStr) - if err != nil { - log.Println("err while parsing hours in timezone offset:", err) - return -1 - } - mins, err := strconv.Atoi(minsStr) - if err != nil { - log.Println("err while parsing mins in timezone offset:", err) - return -1 - } - secs := ((hours * 60) + mins) * 60 - return float64(secs) -} diff --git a/pkg/histanal/histeval.go b/pkg/histanal/histeval.go deleted file mode 100644 index 4d19779..0000000 --- a/pkg/histanal/histeval.go +++ /dev/null @@ -1,246 +0,0 @@ -package histanal - -import ( - "bytes" - "encoding/json" - "fmt" - "log" - "math/rand" - "os" - "os/exec" - - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/strat" - "github.com/jpillora/longestcommon" - - "github.com/schollz/progressbar" -) - -type matchJSON struct { - Match bool - Distance int - CharsRecalled int -} - -type multiMatchItemJSON struct { - Distance int - CharsRecalled int -} - -type multiMatchJSON struct { - Match bool - Entries []multiMatchItemJSON -} - -type strategyJSON struct { - Title string - Description string - Matches []matchJSON - PrefixMatches []multiMatchJSON -} - -// HistEval evaluates history -type HistEval struct { - HistLoad - BatchMode bool - maxCandidates int - Strategies []strategyJSON -} - -// NewHistEval constructs new HistEval -func NewHistEval(inputPath string, - maxCandidates int, skipFailedCmds bool, - debugRecords float64, sanitizedInput bool) HistEval { - - e := HistEval{ - HistLoad: HistLoad{ - skipFailedCmds: skipFailedCmds, - debugRecords: debugRecords, - sanitizedInput: sanitizedInput, - }, - maxCandidates: maxCandidates, - BatchMode: false, - } - records := e.loadHistoryRecords(inputPath) - device := deviceRecords{Records: records} - user := userRecords{} - user.Devices = append(user.Devices, device) - e.UsersRecords = append(e.UsersRecords, user) - e.preprocessRecords() - return e -} - -// NewHistEvalBatchMode constructs new HistEval in batch mode -func NewHistEvalBatchMode(input string, inputDataRoot string, - maxCandidates int, skipFailedCmds bool, - debugRecords float64, sanitizedInput bool) HistEval { - - e := HistEval{ - HistLoad: HistLoad{ - skipFailedCmds: skipFailedCmds, - debugRecords: debugRecords, - sanitizedInput: sanitizedInput, - }, - maxCandidates: maxCandidates, - BatchMode: false, - } - e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) - e.preprocessRecords() - return e -} - -func (e *HistEval) preprocessDeviceRecords(device deviceRecords) deviceRecords { - sessionIDs := map[string]uint64{} - var nextID uint64 - nextID = 1 // start with 1 because 0 won't get saved to json - for k, record := range device.Records { - id, found := sessionIDs[record.SessionID] - if found == false { - id = nextID - sessionIDs[record.SessionID] = id - nextID++ - } - device.Records[k].SeqSessionID = id - // assert - if record.Sanitized != e.sanitizedInput { - if e.sanitizedInput { - log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") - } - log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") - } - device.Records[k].SeqSessionID = id - if e.debugRecords > 0 && rand.Float64() < e.debugRecords { - device.Records[k].DebugThisRecord = true - } - } - // sort.SliceStable(device.Records, func(x, y int) bool { - // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { - // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal - // } - // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID - // }) - - // iterate from back and mark last record of each session - sessionIDSet := map[string]bool{} - for i := len(device.Records) - 1; i >= 0; i-- { - var record *records.EnrichedRecord - record = &device.Records[i] - if sessionIDSet[record.SessionID] { - continue - } - sessionIDSet[record.SessionID] = true - record.LastRecordOfSession = true - } - return device -} - -// enrich records and add sequential session ID -func (e *HistEval) preprocessRecords() { - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) - } - } -} - -// Evaluate a given strategy -func (e *HistEval) Evaluate(strategy strat.IStrategy) error { - title, description := strategy.GetTitleAndDescription() - log.Println("Evaluating strategy:", title, "-", description) - strategyData := strategyJSON{Title: title, Description: description} - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) - var prevRecord records.EnrichedRecord - for _, record := range e.UsersRecords[i].Devices[j].Records { - if e.skipFailedCmds && record.ExitCode != 0 { - continue - } - candidates := strategy.GetCandidates(records.Stripped(record)) - if record.DebugThisRecord { - log.Println() - log.Println("===================================================") - log.Println("STRATEGY:", title, "-", description) - log.Println("===================================================") - log.Println("Previous record:") - if prevRecord.RealtimeBefore == 0 { - log.Println("== NIL") - } else { - rec, _ := prevRecord.ToString() - log.Println(rec) - } - log.Println("---------------------------------------------------") - log.Println("Recommendations for:") - rec, _ := record.ToString() - log.Println(rec) - log.Println("---------------------------------------------------") - for i, candidate := range candidates { - if i > 10 { - break - } - log.Println(string(candidate)) - } - log.Println("===================================================") - } - - matchFound := false - longestPrefixMatchLength := 0 - multiMatch := multiMatchJSON{} - for i, candidate := range candidates { - // make an option (--calculate-total) to turn this on/off ? - // if i >= e.maxCandidates { - // break - // } - commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine})) - if commonPrefixLength > longestPrefixMatchLength { - longestPrefixMatchLength = commonPrefixLength - prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength} - multiMatch.Match = true - multiMatch.Entries = append(multiMatch.Entries, prefixMatch) - } - if candidate == record.CmdLine { - match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} - matchFound = true - strategyData.Matches = append(strategyData.Matches, match) - strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) - break - } - } - if matchFound == false { - strategyData.Matches = append(strategyData.Matches, matchJSON{}) - strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) - } - err := strategy.AddHistoryRecord(&record) - if err != nil { - log.Println("Error while evauating", err) - return err - } - bar.Add(1) - prevRecord = record - } - strategy.ResetHistory() - fmt.Println() - } - } - e.Strategies = append(e.Strategies, strategyData) - return nil -} - -// CalculateStatsAndPlot results -func (e *HistEval) CalculateStatsAndPlot(scriptName string) { - evalJSON, err := json.Marshal(e) - if err != nil { - log.Fatal("json marshal error", err) - } - buffer := bytes.Buffer{} - buffer.Write(evalJSON) - // run python script to stat and plot/ - cmd := exec.Command(scriptName) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = &buffer - err = cmd.Run() - if err != nil { - log.Printf("Command finished with error: %v", err) - } -} diff --git a/pkg/histanal/histload.go b/pkg/histanal/histload.go deleted file mode 100644 index ec81cc2..0000000 --- a/pkg/histanal/histload.go +++ /dev/null @@ -1,180 +0,0 @@ -package histanal - -import ( - "bufio" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "math/rand" - "os" - "path/filepath" - - "github.com/curusarn/resh/pkg/records" -) - -type deviceRecords struct { - Name string - Records []records.EnrichedRecord -} - -type userRecords struct { - Name string - Devices []deviceRecords -} - -// HistLoad loads history -type HistLoad struct { - UsersRecords []userRecords - skipFailedCmds bool - sanitizedInput bool - debugRecords float64 -} - -func (e *HistLoad) preprocessDeviceRecords(device deviceRecords) deviceRecords { - sessionIDs := map[string]uint64{} - var nextID uint64 - nextID = 1 // start with 1 because 0 won't get saved to json - for k, record := range device.Records { - id, found := sessionIDs[record.SessionID] - if found == false { - id = nextID - sessionIDs[record.SessionID] = id - nextID++ - } - device.Records[k].SeqSessionID = id - // assert - if record.Sanitized != e.sanitizedInput { - if e.sanitizedInput { - log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") - } - log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") - } - device.Records[k].SeqSessionID = id - if e.debugRecords > 0 && rand.Float64() < e.debugRecords { - device.Records[k].DebugThisRecord = true - } - } - // sort.SliceStable(device.Records, func(x, y int) bool { - // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { - // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal - // } - // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID - // }) - - // iterate from back and mark last record of each session - sessionIDSet := map[string]bool{} - for i := len(device.Records) - 1; i >= 0; i-- { - var record *records.EnrichedRecord - record = &device.Records[i] - if sessionIDSet[record.SessionID] { - continue - } - sessionIDSet[record.SessionID] = true - record.LastRecordOfSession = true - } - return device -} - -// enrich records and add sequential session ID -func (e *HistLoad) preprocessRecords() { - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) - } - } -} - -func (e *HistLoad) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords { - var records []userRecords - info, err := os.Stat(dataRootPath) - if err != nil { - log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")") - } - if info.IsDir() == false { - log.Fatal("Error:", dataRootPath, "is not a directory - exiting!") - } - users, err := ioutil.ReadDir(dataRootPath) - if err != nil { - log.Fatal("Could not read directory:", dataRootPath) - } - fmt.Println("Listing users in <", dataRootPath, ">...") - for _, user := range users { - userRecords := userRecords{Name: user.Name()} - userFullPath := filepath.Join(dataRootPath, user.Name()) - if user.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.") - continue - } - fmt.Println() - fmt.Printf("*- %s\n", user.Name()) - devices, err := ioutil.ReadDir(userFullPath) - if err != nil { - log.Fatal("Could not read directory:", userFullPath) - } - for _, device := range devices { - deviceRecords := deviceRecords{Name: device.Name()} - deviceFullPath := filepath.Join(userFullPath, device.Name()) - if device.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.") - continue - } - fmt.Printf(" \\- %s\n", device.Name()) - files, err := ioutil.ReadDir(deviceFullPath) - if err != nil { - log.Fatal("Could not read directory:", deviceFullPath) - } - for _, file := range files { - fileFullPath := filepath.Join(deviceFullPath, file.Name()) - if file.Name() == fname { - fmt.Printf(" \\- %s - loading ...", file.Name()) - // load the data - deviceRecords.Records = e.loadHistoryRecords(fileFullPath) - fmt.Println(" OK ✓") - } else { - fmt.Printf(" \\- %s - skipped\n", file.Name()) - } - } - userRecords.Devices = append(userRecords.Devices, deviceRecords) - } - records = append(records, userRecords) - } - return records -} - -func (e *HistLoad) loadHistoryRecords(fname string) []records.EnrichedRecord { - file, err := os.Open(fname) - if err != nil { - log.Fatal("Open() resh history file error:", err) - } - defer file.Close() - - var recs []records.EnrichedRecord - scanner := bufio.NewScanner(file) - for scanner.Scan() { - record := records.Record{} - fallbackRecord := records.FallbackRecord{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) - } - record = records.Convert(&fallbackRecord) - } - if e.sanitizedInput == false { - if record.CmdLength != 0 { - log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?") - } - record.CmdLength = len(record.CmdLine) - } else if record.CmdLength == 0 { - log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") - } - if !e.skipFailedCmds || record.ExitCode == 0 { - recs = append(recs, records.Enriched(record)) - } - } - return recs -} diff --git a/pkg/histcli/histcli.go b/pkg/histcli/histcli.go deleted file mode 100644 index f9b6611..0000000 --- a/pkg/histcli/histcli.go +++ /dev/null @@ -1,31 +0,0 @@ -package histcli - -import ( - "github.com/curusarn/resh/pkg/records" -) - -// Histcli is a dump of history preprocessed for resh cli purposes -type Histcli struct { - // list of records - List []records.CliRecord -} - -// New Histcli -func New() Histcli { - return Histcli{} -} - -// AddRecord to the histcli -func (h *Histcli) AddRecord(record records.Record) { - enriched := records.Enriched(record) - cli := records.NewCliRecord(enriched) - - h.List = append(h.List, cli) -} - -// AddCmdLine to the histcli -func (h *Histcli) AddCmdLine(cmdline string) { - cli := records.NewCliRecordFromCmdLine(cmdline) - - h.List = append(h.List, cli) -} diff --git a/pkg/histfile/histfile.go b/pkg/histfile/histfile.go deleted file mode 100644 index 73e5309..0000000 --- a/pkg/histfile/histfile.go +++ /dev/null @@ -1,262 +0,0 @@ -package histfile - -import ( - "encoding/json" - "log" - "math" - "os" - "strconv" - "sync" - - "github.com/curusarn/resh/pkg/histcli" - "github.com/curusarn/resh/pkg/histlist" - "github.com/curusarn/resh/pkg/records" -) - -// Histfile writes records to histfile -type Histfile struct { - sessionsMutex sync.Mutex - sessions map[string]records.Record - historyPath string - - recentMutex sync.Mutex - recentRecords []records.Record - - // NOTE: we have separate histories which only differ if there was not enough resh_history - // resh_history itself is common for both bash and zsh - bashCmdLines histlist.Histlist - zshCmdLines histlist.Histlist - - cliRecords histcli.Histcli -} - -// New creates new histfile and runs its gorutines -func New(input chan records.Record, sessionsToDrop chan string, - reshHistoryPath string, bashHistoryPath string, zshHistoryPath string, - maxInitHistSize int, minInitHistSizeKB int, - signals chan os.Signal, shutdownDone chan string) *Histfile { - - hf := Histfile{ - sessions: map[string]records.Record{}, - historyPath: reshHistoryPath, - bashCmdLines: histlist.New(), - zshCmdLines: histlist.New(), - cliRecords: histcli.New(), - } - go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB) - go hf.writer(input, signals, shutdownDone) - go hf.sessionGC(sessionsToDrop) - return &hf -} - -// load records from resh history, reverse, enrich and save -func (h *Histfile) loadCliRecords(recs []records.Record) { - for _, cmdline := range h.bashCmdLines.List { - h.cliRecords.AddCmdLine(cmdline) - } - for _, cmdline := range h.zshCmdLines.List { - h.cliRecords.AddCmdLine(cmdline) - } - for i := len(recs) - 1; i >= 0; i-- { - rec := recs[i] - h.cliRecords.AddRecord(rec) - } - log.Println("histfile: resh history loaded - history records count:", len(h.cliRecords.List)) -} - -// loadsHistory from resh_history and if there is not enough of it also load native shell histories -func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHistSize, minInitHistSizeKB int) { - h.recentMutex.Lock() - defer h.recentMutex.Unlock() - log.Println("histfile: Checking if resh_history is large enough ...") - fi, err := os.Stat(h.historyPath) - var size int - if err != nil { - log.Println("histfile ERROR: failed to stat resh_history file:", err) - } else { - size = int(fi.Size()) - } - useNativeHistories := false - if size/1024 < minInitHistSizeKB { - useNativeHistories = true - log.Println("histfile WARN: resh_history is too small - loading native bash and zsh history ...") - h.bashCmdLines = records.LoadCmdLinesFromBashFile(bashHistoryPath) - log.Println("histfile: bash history loaded - cmdLine count:", len(h.bashCmdLines.List)) - h.zshCmdLines = records.LoadCmdLinesFromZshFile(zshHistoryPath) - log.Println("histfile: zsh history loaded - cmdLine count:", len(h.zshCmdLines.List)) - // no maxInitHistSize when using native histories - maxInitHistSize = math.MaxInt32 - } - log.Println("histfile: Loading resh history from file ...") - history := records.LoadFromFile(h.historyPath, math.MaxInt32) - log.Println("histfile: resh history loaded from file - count:", len(history)) - go h.loadCliRecords(history) - // NOTE: keeping this weird interface for now because we might use it in the future - // when we only load bash or zsh history - reshCmdLines := loadCmdLines(history) - log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List)) - if useNativeHistories == false { - h.bashCmdLines = reshCmdLines - h.zshCmdLines = histlist.Copy(reshCmdLines) - return - } - h.bashCmdLines.AddHistlist(reshCmdLines) - log.Println("histfile: bash history + resh history - cmdLine count:", len(h.bashCmdLines.List)) - h.zshCmdLines.AddHistlist(reshCmdLines) - log.Println("histfile: zsh history + resh history - cmdLine count:", len(h.zshCmdLines.List)) -} - -// sessionGC reads sessionIDs from channel and deletes them from histfile struct -func (h *Histfile) sessionGC(sessionsToDrop chan string) { - for { - func() { - session := <-sessionsToDrop - log.Println("histfile: got session to drop", session) - h.sessionsMutex.Lock() - defer h.sessionsMutex.Unlock() - if part1, found := h.sessions[session]; found == true { - log.Println("histfile: Dropping session:", session) - delete(h.sessions, session) - go writeRecord(part1, h.historyPath) - } else { - log.Println("histfile: No hanging parts for session:", session) - } - }() - } -} - -// writer reads records from channel, merges them and writes them to file -func (h *Histfile) writer(input chan records.Record, signals chan os.Signal, shutdownDone chan string) { - for { - func() { - select { - case record := <-input: - h.sessionsMutex.Lock() - defer h.sessionsMutex.Unlock() - - // allows nested sessions to merge records properly - mergeID := record.SessionID + "_" + strconv.Itoa(record.Shlvl) - if record.PartOne { - if _, found := h.sessions[mergeID]; found { - log.Println("histfile WARN: Got another first part of the records before merging the previous one - overwriting! " + - "(this happens in bash because bash-preexec runs when it's not supposed to)") - } - h.sessions[mergeID] = record - } else { - if part1, found := h.sessions[mergeID]; found == false { - log.Println("histfile ERROR: Got second part of records and nothing to merge it with - ignoring! (mergeID:", mergeID, ")") - } else { - delete(h.sessions, mergeID) - go h.mergeAndWriteRecord(part1, record) - } - } - case sig := <-signals: - log.Println("histfile: Got signal " + sig.String()) - h.sessionsMutex.Lock() - defer h.sessionsMutex.Unlock() - log.Println("histfile DEBUG: Unlocked mutex") - - for sessID, record := range h.sessions { - log.Printf("histfile WARN: Writing incomplete record for session: %v\n", sessID) - h.writeRecord(record) - } - log.Println("histfile DEBUG: Shutdown success") - shutdownDone <- "histfile" - return - } - }() - } -} - -func (h *Histfile) writeRecord(part1 records.Record) { - writeRecord(part1, h.historyPath) -} - -func (h *Histfile) mergeAndWriteRecord(part1, part2 records.Record) { - err := part1.Merge(part2) - if err != nil { - log.Println("Error while merging", err) - return - } - - func() { - h.recentMutex.Lock() - defer h.recentMutex.Unlock() - h.recentRecords = append(h.recentRecords, part1) - cmdLine := part1.CmdLine - h.bashCmdLines.AddCmdLine(cmdLine) - h.zshCmdLines.AddCmdLine(cmdLine) - h.cliRecords.AddRecord(part1) - }() - - writeRecord(part1, h.historyPath) -} - -func writeRecord(rec records.Record, outputPath string) { - recJSON, err := json.Marshal(rec) - if err != nil { - log.Println("Marshalling error", err) - return - } - f, err := os.OpenFile(outputPath, - os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Println("Could not open file", err) - return - } - defer f.Close() - _, err = f.Write(append(recJSON, []byte("\n")...)) - if err != nil { - log.Printf("Error while writing: %v, %s\n", rec, err) - return - } -} - -// GetRecentCmdLines returns recent cmdLines -func (h *Histfile) GetRecentCmdLines(shell string, limit int) histlist.Histlist { - // NOTE: limit does nothing atm - h.recentMutex.Lock() - defer h.recentMutex.Unlock() - log.Println("histfile: History requested ...") - var hl histlist.Histlist - if shell == "bash" { - hl = histlist.Copy(h.bashCmdLines) - log.Println("histfile: history copied (bash) - cmdLine count:", len(hl.List)) - return hl - } - if shell != "zsh" { - log.Println("histfile ERROR: Unknown shell: ", shell) - } - hl = histlist.Copy(h.zshCmdLines) - log.Println("histfile: history copied (zsh) - cmdLine count:", len(hl.List)) - return hl -} - -// DumpCliRecords returns enriched records -func (h *Histfile) DumpCliRecords() histcli.Histcli { - // don't forget locks in the future - return h.cliRecords -} - -func loadCmdLines(recs []records.Record) histlist.Histlist { - hl := histlist.New() - // go from bottom and deduplicate - var cmdLines []string - cmdLinesSet := map[string]bool{} - for i := len(recs) - 1; i >= 0; i-- { - cmdLine := recs[i].CmdLine - if cmdLinesSet[cmdLine] { - continue - } - cmdLinesSet[cmdLine] = true - cmdLines = append([]string{cmdLine}, cmdLines...) - // if len(cmdLines) > limit { - // break - // } - } - // add everything to histlist - for _, cmdLine := range cmdLines { - hl.AddCmdLine(cmdLine) - } - return hl -} diff --git a/pkg/msg/msg.go b/pkg/msg/msg.go deleted file mode 100644 index 3c85987..0000000 --- a/pkg/msg/msg.go +++ /dev/null @@ -1,32 +0,0 @@ -package msg - -import "github.com/curusarn/resh/pkg/records" - -// CliMsg struct -type CliMsg struct { - SessionID string `json:"sessionID"` - PWD string `json:"pwd"` -} - -// CliResponse struct -type CliResponse struct { - CliRecords []records.CliRecord `json:"cliRecords"` -} - -// InspectMsg struct -type InspectMsg struct { - SessionID string `json:"sessionId"` - Count uint `json:"count"` -} - -// MultiResponse struct -type MultiResponse struct { - CmdLines []string `json:"cmdlines"` -} - -// StatusResponse struct -type StatusResponse struct { - Status bool `json:"status"` - Version string `json:"version"` - Commit string `json:"commit"` -} diff --git a/pkg/records/records.go b/pkg/records/records.go deleted file mode 100644 index 6271ba5..0000000 --- a/pkg/records/records.go +++ /dev/null @@ -1,689 +0,0 @@ -package records - -import ( - "bufio" - "encoding/json" - "errors" - "io" - "log" - "math" - "os" - "strconv" - "strings" - - "github.com/curusarn/resh/pkg/histlist" - "github.com/mattn/go-shellwords" -) - -// BaseRecord - common base for Record and FallbackRecord -type BaseRecord struct { - // core - CmdLine string `json:"cmdLine"` - ExitCode int `json:"exitCode"` - Shell string `json:"shell"` - Uname string `json:"uname"` - SessionID string `json:"sessionId"` - RecordID string `json:"recordId"` - - // posix - Home string `json:"home"` - Lang string `json:"lang"` - LcAll string `json:"lcAll"` - Login string `json:"login"` - //Path string `json:"path"` - Pwd string `json:"pwd"` - PwdAfter string `json:"pwdAfter"` - ShellEnv string `json:"shellEnv"` - Term string `json:"term"` - - // non-posix"` - RealPwd string `json:"realPwd"` - RealPwdAfter string `json:"realPwdAfter"` - Pid int `json:"pid"` - SessionPID int `json:"sessionPid"` - Host string `json:"host"` - Hosttype string `json:"hosttype"` - Ostype string `json:"ostype"` - Machtype string `json:"machtype"` - Shlvl int `json:"shlvl"` - - // before after - TimezoneBefore string `json:"timezoneBefore"` - TimezoneAfter string `json:"timezoneAfter"` - - RealtimeBefore float64 `json:"realtimeBefore"` - RealtimeAfter float64 `json:"realtimeAfter"` - RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` - RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` - - RealtimeDuration float64 `json:"realtimeDuration"` - RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` - RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` - //Logs []string `json: "logs"` - - GitDir string `json:"gitDir"` - GitRealDir string `json:"gitRealDir"` - GitOriginRemote string `json:"gitOriginRemote"` - GitDirAfter string `json:"gitDirAfter"` - GitRealDirAfter string `json:"gitRealDirAfter"` - GitOriginRemoteAfter string `json:"gitOriginRemoteAfter"` - MachineID string `json:"machineId"` - - OsReleaseID string `json:"osReleaseId"` - OsReleaseVersionID string `json:"osReleaseVersionId"` - OsReleaseIDLike string `json:"osReleaseIdLike"` - OsReleaseName string `json:"osReleaseName"` - OsReleasePrettyName string `json:"osReleasePrettyName"` - - ReshUUID string `json:"reshUuid"` - ReshVersion string `json:"reshVersion"` - ReshRevision string `json:"reshRevision"` - - // records come in two parts (collect and postcollect) - PartOne bool `json:"partOne,omitempty"` // false => part two - PartsMerged bool `json:"partsMerged"` - // special flag -> not an actual record but an session end - SessionExit bool `json:"sessionExit,omitempty"` - - // recall metadata - Recalled bool `json:"recalled"` - RecallHistno int `json:"recallHistno,omitempty"` - RecallStrategy string `json:"recallStrategy,omitempty"` - RecallActionsRaw string `json:"recallActionsRaw,omitempty"` - RecallActions []string `json:"recallActions,omitempty"` - RecallLastCmdLine string `json:"recallLastCmdLine"` - - // recall command - RecallPrefix string `json:"recallPrefix,omitempty"` - - // added by sanitizatizer - Sanitized bool `json:"sanitized,omitempty"` - CmdLength int `json:"cmdLength,omitempty"` -} - -// Record representing single executed command with its metadata -type Record struct { - BaseRecord - - Cols string `json:"cols"` - Lines string `json:"lines"` -} - -// EnrichedRecord - record enriched with additional data -type EnrichedRecord struct { - Record - - // enriching fields - added "later" - Command string `json:"command"` - FirstWord string `json:"firstWord"` - Invalid bool `json:"invalid"` - SeqSessionID uint64 `json:"seqSessionId"` - LastRecordOfSession bool `json:"lastRecordOfSession"` - DebugThisRecord bool `json:"debugThisRecord"` - Errors []string `json:"errors"` - // SeqSessionID uint64 `json:"seqSessionId,omitempty"` -} - -// FallbackRecord when record is too old and can't be parsed into regular Record -type FallbackRecord struct { - BaseRecord - // older version of the record where cols and lines are int - - Cols int `json:"cols"` // notice the int type - Lines int `json:"lines"` // notice the int type -} - -// SlimRecord used for recalling because unmarshalling record w/ 50+ fields is too slow -type SlimRecord struct { - SessionID string `json:"sessionId"` - RecallHistno int `json:"recallHistno,omitempty"` - RecallPrefix string `json:"recallPrefix,omitempty"` - - // extra recall - we might use these in the future - // Pwd string `json:"pwd"` - // RealPwd string `json:"realPwd"` - // GitDir string `json:"gitDir"` - // GitRealDir string `json:"gitRealDir"` - // GitOriginRemote string `json:"gitOriginRemote"` - -} - -// CliRecord used for sending records to RESH-CLI -type CliRecord struct { - IsRaw bool `json:"isRaw"` - SessionID string `json:"sessionId"` - - CmdLine string `json:"cmdLine"` - Host string `json:"host"` - Pwd string `json:"pwd"` - Home string `json:"home"` // helps us to collapse /home/user to tilde - GitOriginRemote string `json:"gitOriginRemote"` - ExitCode int `json:"exitCode"` - - RealtimeBefore float64 `json:"realtimeBefore"` - // RealtimeAfter float64 `json:"realtimeAfter"` - // RealtimeDuration float64 `json:"realtimeDuration"` -} - -// NewCliRecordFromCmdLine from EnrichedRecord -func NewCliRecordFromCmdLine(cmdLine string) CliRecord { - return CliRecord{ - IsRaw: true, - CmdLine: cmdLine, - } -} - -// NewCliRecord from EnrichedRecord -func NewCliRecord(r EnrichedRecord) CliRecord { - return CliRecord{ - IsRaw: false, - SessionID: r.SessionID, - CmdLine: r.CmdLine, - Host: r.Host, - Pwd: r.Pwd, - Home: r.Home, - GitOriginRemote: r.GitOriginRemote, - ExitCode: r.ExitCode, - RealtimeBefore: r.RealtimeBefore, - } -} - -// Convert from FallbackRecord to Record -func Convert(r *FallbackRecord) Record { - return Record{ - BaseRecord: r.BaseRecord, - // these two lines are the only reason we are doing this - Cols: strconv.Itoa(r.Cols), - Lines: strconv.Itoa(r.Lines), - } -} - -// ToString - returns record the json -func (r EnrichedRecord) ToString() (string, error) { - jsonRec, err := json.Marshal(r) - if err != nil { - return "marshalling error", err - } - return string(jsonRec), nil -} - -// Enriched - returnd enriched record -func Enriched(r Record) EnrichedRecord { - record := EnrichedRecord{Record: r} - // normlize git remote - record.GitOriginRemote = NormalizeGitRemote(record.GitOriginRemote) - record.GitOriginRemoteAfter = NormalizeGitRemote(record.GitOriginRemoteAfter) - // Get command/first word from commandline - var err error - err = r.Validate() - if err != nil { - record.Errors = append(record.Errors, "Validate error:"+err.Error()) - // rec, _ := record.ToString() - // log.Println("Invalid command:", rec) - record.Invalid = true - } - record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine) - if err != nil { - record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error()) - // rec, _ := record.ToString() - // log.Println("Invalid command:", rec) - record.Invalid = true // should this be really invalid ? - } - return record -} - -// Merge two records (part1 - collect + part2 - postcollect) -func (r *Record) Merge(r2 Record) error { - if r.PartOne == false || r2.PartOne { - return errors.New("Expected part1 and part2 of the same record - usage: part1.Merge(part2)") - } - if r.SessionID != r2.SessionID { - return errors.New("Records to merge are not from the same sesion - r1:" + r.SessionID + " r2:" + r2.SessionID) - } - if r.CmdLine != r2.CmdLine { - return errors.New("Records to merge are not parts of the same records - r1:" + r.CmdLine + " r2:" + r2.CmdLine) - } - if r.RecordID != r2.RecordID { - return errors.New("Records to merge do not have the same ID - r1:" + r.RecordID + " r2:" + r2.RecordID) - } - // r.RealtimeBefore != r2.RealtimeBefore - can't be used because of bash-preexec runs when it's not supposed to - r.ExitCode = r2.ExitCode - r.PwdAfter = r2.PwdAfter - r.RealPwdAfter = r2.RealPwdAfter - r.GitDirAfter = r2.GitDirAfter - r.GitRealDirAfter = r2.GitRealDirAfter - r.RealtimeAfter = r2.RealtimeAfter - r.GitOriginRemoteAfter = r2.GitOriginRemoteAfter - r.TimezoneAfter = r2.TimezoneAfter - r.RealtimeAfterLocal = r2.RealtimeAfterLocal - r.RealtimeDuration = r2.RealtimeDuration - - r.PartsMerged = true - r.PartOne = false - return nil -} - -// Validate - returns error if the record is invalid -func (r *Record) Validate() error { - if r.CmdLine == "" { - return errors.New("There is no CmdLine") - } - if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 { - return errors.New("There is no Time") - } - if r.RealtimeBeforeLocal == 0 || r.RealtimeAfterLocal == 0 { - return errors.New("There is no Local Time") - } - if r.RealPwd == "" || r.RealPwdAfter == "" { - return errors.New("There is no Real Pwd") - } - if r.Pwd == "" || r.PwdAfter == "" { - return errors.New("There is no Pwd") - } - - // TimezoneBefore - // TimezoneAfter - - // RealtimeDuration - // RealtimeSinceSessionStart - TODO: add later - // RealtimeSinceBoot - TODO: add later - - // device extras - // Host - // Hosttype - // Ostype - // Machtype - // OsReleaseID - // OsReleaseVersionID - // OsReleaseIDLike - // OsReleaseName - // OsReleasePrettyName - - // session extras - // Term - // Shlvl - - // static info - // Lang - // LcAll - - // meta - // ReshUUID - // ReshVersion - // ReshRevision - - // added by sanitizatizer - // Sanitized - // CmdLength - return nil -} - -// SetCmdLine sets cmdLine and related members -func (r *EnrichedRecord) SetCmdLine(cmdLine string) { - r.CmdLine = cmdLine - r.CmdLength = len(cmdLine) - r.ExitCode = 0 - var err error - r.Command, r.FirstWord, err = GetCommandAndFirstWord(cmdLine) - if err != nil { - r.Errors = append(r.Errors, "GetCommandAndFirstWord error:"+err.Error()) - // log.Println("Invalid command:", r.CmdLine) - r.Invalid = true - } -} - -// Stripped returns record stripped of all info that is not available during prediction -func Stripped(r EnrichedRecord) EnrichedRecord { - // clear the cmd itself - r.SetCmdLine("") - // replace after info with before info - r.PwdAfter = r.Pwd - r.RealPwdAfter = r.RealPwd - r.TimezoneAfter = r.TimezoneBefore - r.RealtimeAfter = r.RealtimeBefore - r.RealtimeAfterLocal = r.RealtimeBeforeLocal - // clear some more stuff - r.RealtimeDuration = 0 - r.LastRecordOfSession = false - return r -} - -// GetCommandAndFirstWord func -func GetCommandAndFirstWord(cmdLine string) (string, string, error) { - args, err := shellwords.Parse(cmdLine) - if err != nil { - // log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") - return "", "", err - } - if len(args) == 0 { - return "", "", nil - } - i := 0 - for true { - // commands in shell sometimes look like this `variable=something command argument otherArgument --option` - // to get the command we skip over tokens that contain '=' - if strings.ContainsRune(args[i], '=') && len(args) > i+1 { - i++ - continue - } - return args[i], args[0], nil - } - log.Fatal("GetCommandAndFirstWord error: this should not happen!") - return "ERROR", "ERROR", errors.New("this should not happen - contact developer ;)") -} - -// NormalizeGitRemote func -func NormalizeGitRemote(gitRemote string) string { - if strings.HasSuffix(gitRemote, ".git") { - return gitRemote[:len(gitRemote)-4] - } - return gitRemote -} - -// DistParams is used to supply params to Enrichedrecords.DistanceTo() -type DistParams struct { - ExitCode float64 - MachineID float64 - SessionID float64 - Login float64 - Shell float64 - Pwd float64 - RealPwd float64 - Git float64 - Time float64 -} - -// DistanceTo another record -func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { - var dist float64 - dist = 0 - - // lev distance or something? TODO later - // CmdLine - - // exit code - if r.ExitCode != r2.ExitCode { - if r.ExitCode == 0 || r2.ExitCode == 0 { - // one success + one error -> 1 - dist += 1 * p.ExitCode - } else { - // two different errors - dist += 0.5 * p.ExitCode - } - } - - // machine/device - if r.MachineID != r2.MachineID { - dist += 1 * p.MachineID - } - // Uname - - // session - if r.SessionID != r2.SessionID { - dist += 1 * p.SessionID - } - // Pid - add because of nested shells? - // SessionPid - - // user - if r.Login != r2.Login { - dist += 1 * p.Login - } - // Home - - // shell - if r.Shell != r2.Shell { - dist += 1 * p.Shell - } - // ShellEnv - - // pwd - if r.Pwd != r2.Pwd { - // TODO: compare using hierarchy - // TODO: make more important - dist += 1 * p.Pwd - } - if r.RealPwd != r2.RealPwd { - // TODO: -||- - dist += 1 * p.RealPwd - } - // PwdAfter - // RealPwdAfter - - // git - if r.GitDir != r2.GitDir { - dist += 1 * p.Git - } - if r.GitRealDir != r2.GitRealDir { - dist += 1 * p.Git - } - if r.GitOriginRemote != r2.GitOriginRemote { - dist += 1 * p.Git - } - - // time - // this can actually get negative for differences of less than one second which is fine - // distance grows by 1 with every order - distTime := math.Log10(math.Abs(r.RealtimeBefore-r2.RealtimeBefore)) * p.Time - if math.IsNaN(distTime) == false && math.IsInf(distTime, 0) == false { - dist += distTime - } - // RealtimeBeforeLocal - // RealtimeAfter - // RealtimeAfterLocal - - // TimezoneBefore - // TimezoneAfter - - // RealtimeDuration - // RealtimeSinceSessionStart - TODO: add later - // RealtimeSinceBoot - TODO: add later - - // device extras - // Host - // Hosttype - // Ostype - // Machtype - // OsReleaseID - // OsReleaseVersionID - // OsReleaseIDLike - // OsReleaseName - // OsReleasePrettyName - - // session extras - // Term - // Shlvl - - // static info - // Lang - // LcAll - - // meta - // ReshUUID - // ReshVersion - // ReshRevision - - // added by sanitizatizer - // Sanitized - // CmdLength - - return dist -} - -// LoadFromFile loads records from 'fname' file -func LoadFromFile(fname string, limit int) []Record { - const allowedErrors = 1 - var encounteredErrors int - // NOTE: limit does nothing atm - var recs []Record - file, err := os.Open(fname) - if err != nil { - log.Println("Open() resh history file error:", err) - log.Println("WARN: Skipping reading resh history!") - return recs - } - defer file.Close() - - reader := bufio.NewReader(file) - var i int - var firstErrLine int - for { - line, err := reader.ReadString('\n') - if err != nil { - break - } - i++ - record := Record{} - fallbackRecord := FallbackRecord{} - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - if encounteredErrors == 0 { - firstErrLine = i - } - encounteredErrors++ - log.Println("Line:", line) - log.Println("Decoding error:", err) - if encounteredErrors > allowedErrors { - log.Fatalf("Fatal: Encountered more than %d decoding errors (%d)", allowedErrors, encounteredErrors) - } - } - record = Convert(&fallbackRecord) - } - recs = append(recs, record) - } - // log.Println("records: done loading file:", err) - if err != io.EOF { - log.Println("records: error while loading file:", err) - } - // log.Println("records: Loaded lines - count:", i) - if encounteredErrors > 0 { - // fix errors in the history file - log.Printf("There were %d decoding errors, the first error happend on line %d/%d", encounteredErrors, firstErrLine, i) - log.Println("Backing up current history file ...") - err := copyFile(fname, fname+".bak") - if err != nil { - log.Fatalln("Failed to backup history file with decode errors") - } - log.Println("Writing out a history file without errors ...") - err = writeHistory(fname, recs) - if err != nil { - log.Fatalln("Fatal: Failed write out new history") - } - } - log.Println("records: Loaded records - count:", len(recs)) - return recs -} - -func copyFile(source, dest string) error { - from, err := os.Open(source) - if err != nil { - // log.Println("Open() resh history file error:", err) - return err - } - defer from.Close() - - // to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666) - to, err := os.Create(dest) - if err != nil { - // log.Println("Create() resh history backup error:", err) - return err - } - defer to.Close() - - _, err = io.Copy(to, from) - if err != nil { - // log.Println("Copy() resh history to backup error:", err) - return err - } - return nil -} - -func writeHistory(fname string, history []Record) error { - file, err := os.Create(fname) - if err != nil { - // log.Println("Create() resh history error:", err) - return err - } - defer file.Close() - for _, rec := range history { - jsn, err := json.Marshal(rec) - if err != nil { - log.Fatalln("Encode error!") - } - file.Write(append(jsn, []byte("\n")...)) - } - return nil -} - -// LoadCmdLinesFromZshFile loads cmdlines from zsh history file -func LoadCmdLinesFromZshFile(fname string) histlist.Histlist { - hl := histlist.New() - file, err := os.Open(fname) - if err != nil { - log.Println("Open() zsh history file error:", err) - log.Println("WARN: Skipping reading zsh history!") - return hl - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - // trim newline - line = strings.TrimRight(line, "\n") - var cmd string - // zsh format EXTENDED_HISTORY - // : 1576270617:0;make install - // zsh format no EXTENDED_HISTORY - // make install - if len(line) == 0 { - // skip empty - continue - } - if strings.Contains(line, ":") && strings.Contains(line, ";") && - len(strings.Split(line, ":")) >= 3 && len(strings.Split(line, ";")) >= 2 { - // contains at least 2x ':' and 1x ';' => assume EXTENDED_HISTORY - cmd = strings.Split(line, ";")[1] - } else { - cmd = line - } - hl.AddCmdLine(cmd) - } - return hl -} - -// LoadCmdLinesFromBashFile loads cmdlines from bash history file -func LoadCmdLinesFromBashFile(fname string) histlist.Histlist { - hl := histlist.New() - file, err := os.Open(fname) - if err != nil { - log.Println("Open() bash history file error:", err) - log.Println("WARN: Skipping reading bash history!") - return hl - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - // trim newline - line = strings.TrimRight(line, "\n") - // trim spaces from left - line = strings.TrimLeft(line, " ") - // bash format (two lines) - // #1576199174 - // make install - if strings.HasPrefix(line, "#") { - // is either timestamp or comment => skip - continue - } - if len(line) == 0 { - // skip empty - continue - } - hl.AddCmdLine(line) - } - return hl -} diff --git a/pkg/records/records_test.go b/pkg/records/records_test.go deleted file mode 100644 index 5ef3c55..0000000 --- a/pkg/records/records_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package records - -import ( - "bufio" - "encoding/json" - "log" - "os" - "testing" -) - -func GetTestRecords() []Record { - file, err := os.Open("testdata/resh_history.json") - if err != nil { - log.Fatal("Open() resh history file error:", err) - } - defer file.Close() - - var recs []Record - scanner := bufio.NewScanner(file) - for scanner.Scan() { - record := Record{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) - } - recs = append(recs, record) - } - return recs -} - -func GetTestEnrichedRecords() []EnrichedRecord { - var recs []EnrichedRecord - for _, rec := range GetTestRecords() { - recs = append(recs, Enriched(rec)) - } - return recs -} - -func TestToString(t *testing.T) { - for _, rec := range GetTestEnrichedRecords() { - _, err := rec.ToString() - if err != nil { - t.Error("ToString() failed") - } - } -} - -func TestEnriched(t *testing.T) { - record := Record{BaseRecord: BaseRecord{CmdLine: "cmd arg1 arg2"}} - enriched := Enriched(record) - if enriched.FirstWord != "cmd" || enriched.Command != "cmd" { - t.Error("Enriched() returned reocord w/ wrong Command OR FirstWord") - } -} - -func TestValidate(t *testing.T) { - record := EnrichedRecord{} - if record.Validate() == nil { - t.Error("Validate() didn't return an error for invalid record") - } - record.CmdLine = "cmd arg" - record.FirstWord = "cmd" - record.Command = "cmd" - time := 1234.5678 - record.RealtimeBefore = time - record.RealtimeAfter = time - record.RealtimeBeforeLocal = time - record.RealtimeAfterLocal = time - pwd := "/pwd" - record.Pwd = pwd - record.PwdAfter = pwd - record.RealPwd = pwd - record.RealPwdAfter = pwd - if record.Validate() != nil { - t.Error("Validate() returned an error for a valid record") - } -} - -func TestSetCmdLine(t *testing.T) { - record := EnrichedRecord{} - cmdline := "cmd arg1 arg2" - record.SetCmdLine(cmdline) - if record.CmdLine != cmdline || record.Command != "cmd" || record.FirstWord != "cmd" { - t.Error() - } -} - -func TestStripped(t *testing.T) { - for _, rec := range GetTestEnrichedRecords() { - stripped := Stripped(rec) - - // there should be no cmdline - if stripped.CmdLine != "" || - stripped.FirstWord != "" || - stripped.Command != "" { - t.Error("Stripped() returned record w/ info about CmdLine, Command OR FirstWord") - } - // *after* fields should be overwritten by *before* fields - if stripped.PwdAfter != stripped.Pwd || - stripped.RealPwdAfter != stripped.RealPwd || - stripped.TimezoneAfter != stripped.TimezoneBefore || - stripped.RealtimeAfter != stripped.RealtimeBefore || - stripped.RealtimeAfterLocal != stripped.RealtimeBeforeLocal { - t.Error("Stripped() returned record w/ different *after* and *before* values - *after* fields should be overwritten by *before* fields") - } - // there should be no information about duration and session end - if stripped.RealtimeDuration != 0 || - stripped.LastRecordOfSession != false { - t.Error("Stripped() returned record with too much information") - } - } -} - -func TestGetCommandAndFirstWord(t *testing.T) { - cmd, stWord, err := GetCommandAndFirstWord("cmd arg1 arg2") - if err != nil || cmd != "cmd" || stWord != "cmd" { - t.Error("GetCommandAndFirstWord() returned wrong Command OR FirstWord") - } -} - -func TestDistanceTo(t *testing.T) { - paramsFull := DistParams{ - ExitCode: 1, - MachineID: 1, - SessionID: 1, - Login: 1, - Shell: 1, - Pwd: 1, - RealPwd: 1, - Git: 1, - Time: 1, - } - paramsZero := DistParams{} - var prevRec EnrichedRecord - for _, rec := range GetTestEnrichedRecords() { - dist := rec.DistanceTo(rec, paramsFull) - if dist != 0 { - t.Error("DistanceTo() itself should be always 0") - } - dist = rec.DistanceTo(prevRec, paramsFull) - if dist == 0 { - t.Error("DistanceTo() between two test records shouldn't be 0") - } - dist = rec.DistanceTo(prevRec, paramsZero) - if dist != 0 { - t.Error("DistanceTo() should be 0 when DistParams is all zeros") - } - prevRec = rec - } -} diff --git a/pkg/records/testdata/resh_history.json b/pkg/records/testdata/resh_history.json deleted file mode 100644 index 40f43ab..0000000 --- a/pkg/records/testdata/resh_history.json +++ /dev/null @@ -1,27 +0,0 @@ -{"cmdLine":"ls","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"d5c0fe70-c80b-4715-87cb-f8d8d5b4c673","cols":"80","lines":"24","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14560,"sessionPid":14560,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1566762905.173595,"realtimeAfter":1566762905.1894295,"realtimeBeforeLocal":1566770105.173595,"realtimeAfterLocal":1566770105.1894295,"realtimeDuration":0.015834569931030273,"realtimeSinceSessionStart":1.7122540473937988,"realtimeSinceBoot":20766.542254047396,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"find . -name applications","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567420001.2531302,"realtimeAfter":1567420002.4311218,"realtimeBeforeLocal":1567427201.2531302,"realtimeAfterLocal":1567427202.4311218,"realtimeDuration":1.1779916286468506,"realtimeSinceSessionStart":957.4848053455353,"realtimeSinceBoot":2336.594805345535,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"desktop-file-validate curusarn.sync-clipboards.desktop ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/.local/share/applications","pwdAfter":"/home/simon/.local/share/applications","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/.local/share/applications","realPwdAfter":"/home/simon/.local/share/applications","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567421748.2965438,"realtimeAfter":1567421748.3068867,"realtimeBeforeLocal":1567428948.2965438,"realtimeAfterLocal":1567428948.3068867,"realtimeDuration":0.010342836380004883,"realtimeSinceSessionStart":2704.528218984604,"realtimeSinceBoot":4083.6382189846036,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"cat /tmp/extensions | grep '.'","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461416.6871984,"realtimeAfter":1567461416.7336714,"realtimeBeforeLocal":1567468616.6871984,"realtimeAfterLocal":1567468616.7336714,"realtimeDuration":0.046473026275634766,"realtimeSinceSessionStart":21.45597553253174,"realtimeSinceBoot":43752.03597553253,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"cd git/resh/","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461667.8806899,"realtimeAfter":1567461667.8949044,"realtimeBeforeLocal":1567468867.8806899,"realtimeAfterLocal":1567468867.8949044,"realtimeDuration":0.014214515686035156,"realtimeSinceSessionStart":272.64946699142456,"realtimeSinceBoot":44003.229466991426,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461707.6467602,"realtimeAfter":1567461707.7177293,"realtimeBeforeLocal":1567468907.6467602,"realtimeAfterLocal":1567468907.7177293,"realtimeDuration":0.0709691047668457,"realtimeSinceSessionStart":312.4155373573303,"realtimeSinceBoot":44042.99553735733,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"cat /tmp/extensions | grep '^\\.' | cut -f1 |tr '[:upper:]' '[:lower:]' ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461722.813049,"realtimeAfter":1567461722.8280325,"realtimeBeforeLocal":1567468922.813049,"realtimeAfterLocal":1567468922.8280325,"realtimeDuration":0.014983415603637695,"realtimeSinceSessionStart":327.581826210022,"realtimeSinceBoot":44058.161826210024,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"tig","exitCode":127,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461906.3896828,"realtimeAfter":1567461906.4084594,"realtimeBeforeLocal":1567469106.3896828,"realtimeAfterLocal":1567469106.4084594,"realtimeDuration":0.018776655197143555,"realtimeSinceSessionStart":511.1584599018097,"realtimeSinceBoot":44241.73845990181,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"resh-sanitize-history | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a3318c80-3521-4b22-aa64-ea0f6c641410","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14601,"sessionPid":14601,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567547116.2430356,"realtimeAfter":1567547116.7547352,"realtimeBeforeLocal":1567554316.2430356,"realtimeAfterLocal":1567554316.7547352,"realtimeDuration":0.5116996765136719,"realtimeSinceSessionStart":15.841878414154053,"realtimeSinceBoot":30527.201878414155,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo pacman -S ansible","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609042.0166302,"realtimeAfter":1567609076.9726007,"realtimeBeforeLocal":1567616242.0166302,"realtimeAfterLocal":1567616276.9726007,"realtimeDuration":34.95597052574158,"realtimeSinceSessionStart":1617.0794131755829,"realtimeSinceBoot":6120.029413175583,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"vagrant up","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609090.7359188,"realtimeAfter":1567609098.3125577,"realtimeBeforeLocal":1567616290.7359188,"realtimeAfterLocal":1567616298.3125577,"realtimeDuration":7.57663893699646,"realtimeSinceSessionStart":1665.798701763153,"realtimeSinceBoot":6168.748701763153,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo modprobe vboxnetflt","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609143.2847652,"realtimeAfter":1567609143.3116078,"realtimeBeforeLocal":1567616343.2847652,"realtimeAfterLocal":1567616343.3116078,"realtimeDuration":0.026842594146728516,"realtimeSinceSessionStart":1718.3475482463837,"realtimeSinceBoot":6221.2975482463835,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"echo $RANDOM","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"8ddacadc-6e73-483c-b347-4e18df204466","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":31387,"sessionPid":31387,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567727039.6540458,"realtimeAfter":1567727039.6629689,"realtimeBeforeLocal":1567734239.6540458,"realtimeAfterLocal":1567734239.6629689,"realtimeDuration":0.008923053741455078,"realtimeSinceSessionStart":1470.7667458057404,"realtimeSinceBoot":18495.01674580574,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"make resh-evaluate ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567977478.9672194,"realtimeAfter":1567977479.5449634,"realtimeBeforeLocal":1567984678.9672194,"realtimeAfterLocal":1567984679.5449634,"realtimeDuration":0.5777440071105957,"realtimeSinceSessionStart":5738.577540636063,"realtimeSinceBoot":20980.42754063606,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"cat ~/.resh_history.json | grep \"./resh-eval\" | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"105","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567986105.3988302,"realtimeAfter":1567986105.4809113,"realtimeBeforeLocal":1567993305.3988302,"realtimeAfterLocal":1567993305.4809113,"realtimeDuration":0.08208107948303223,"realtimeSinceSessionStart":14365.00915145874,"realtimeSinceBoot":29606.85915145874,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git c \"add sanitized flag to record, add Enrich() to record\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063976.9103937,"realtimeAfter":1568063976.9326868,"realtimeBeforeLocal":1568071176.9103937,"realtimeAfterLocal":1568071176.9326868,"realtimeDuration":0.0222930908203125,"realtimeSinceSessionStart":92236.52071499825,"realtimeSinceBoot":107478.37071499825,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063978.2340608,"realtimeAfter":1568063978.252463,"realtimeBeforeLocal":1568071178.2340608,"realtimeAfterLocal":1568071178.252463,"realtimeDuration":0.0184023380279541,"realtimeSinceSessionStart":92237.84438204765,"realtimeSinceBoot":107479.69438204766,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git a evaluate/results.go ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063989.0446353,"realtimeAfter":1568063989.2452207,"realtimeBeforeLocal":1568071189.0446353,"realtimeAfterLocal":1568071189.2452207,"realtimeDuration":0.20058536529541016,"realtimeSinceSessionStart":92248.65495657921,"realtimeSinceBoot":107490.50495657921,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo pacman -S python-pip","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072068.3557143,"realtimeAfter":1568072070.7509863,"realtimeBeforeLocal":1568079268.3557143,"realtimeAfterLocal":1568079270.7509863,"realtimeDuration":2.3952720165252686,"realtimeSinceSessionStart":100327.96603560448,"realtimeSinceBoot":115569.81603560448,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"pip3 install matplotlib","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072088.5575967,"realtimeAfter":1568072094.372314,"realtimeBeforeLocal":1568079288.5575967,"realtimeAfterLocal":1568079294.372314,"realtimeDuration":5.8147172927856445,"realtimeSinceSessionStart":100348.16791796684,"realtimeSinceBoot":115590.01791796685,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo pip3 install matplotlib","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072106.138616,"realtimeAfter":1568072115.1124601,"realtimeBeforeLocal":1568079306.138616,"realtimeAfterLocal":1568079315.1124601,"realtimeDuration":8.973844051361084,"realtimeSinceSessionStart":100365.7489373684,"realtimeSinceBoot":115607.5989373684,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"./resh-evaluate --plotting-script evaluate/resh-evaluate-plot.py --input ~/git/resh_private/history_data/simon/dell/resh_history.json ","exitCode":130,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568076266.9364285,"realtimeAfter":1568076288.1131275,"realtimeBeforeLocal":1568083466.9364285,"realtimeAfterLocal":1568083488.1131275,"realtimeDuration":21.176698923110962,"realtimeSinceSessionStart":104526.54674983025,"realtimeSinceBoot":119768.39674983025,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git c \"Add a bunch of useless comments to make linter happy\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"04050353-a97d-4435-9248-f47dd08b2f2a","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":14702,"sessionPid":14702,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569456045.8763022,"realtimeAfter":1569456045.9030173,"realtimeBeforeLocal":1569463245.8763022,"realtimeAfterLocal":1569463245.9030173,"realtimeDuration":0.02671504020690918,"realtimeSinceSessionStart":2289.789242744446,"realtimeSinceBoot":143217.91924274445,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"fuck","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a4aadf03-610d-4731-ba94-5b7ce21e7bb9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":3413,"sessionPid":3413,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569687682.4250975,"realtimeAfter":1569687682.5877323,"realtimeBeforeLocal":1569694882.4250975,"realtimeAfterLocal":1569694882.5877323,"realtimeDuration":0.16263484954833984,"realtimeSinceSessionStart":264603.49496507645,"realtimeSinceBoot":374854.48496507644,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"code .","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709366.523767,"realtimeAfter":1569709367.516908,"realtimeBeforeLocal":1569716566.523767,"realtimeAfterLocal":1569716567.516908,"realtimeDuration":0.9931409358978271,"realtimeSinceSessionStart":23846.908839941025,"realtimeSinceBoot":396539.888839941,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"make test","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709371.89966,"realtimeAfter":1569709377.430194,"realtimeBeforeLocal":1569716571.89966,"realtimeAfterLocal":1569716577.430194,"realtimeDuration":5.530533790588379,"realtimeSinceSessionStart":23852.284733057022,"realtimeSinceBoot":396545.264733057,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"mkdir ~/git/resh/testdata","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"71529b60-2e7b-4d5b-8dc1-6d0740b58e9e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":21224,"sessionPid":21224,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709838.4642656,"realtimeAfter":1569709838.4718792,"realtimeBeforeLocal":1569717038.4642656,"realtimeAfterLocal":1569717038.4718792,"realtimeDuration":0.007613658905029297,"realtimeSinceSessionStart":9.437154054641724,"realtimeSinceBoot":397011.02715405467,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} diff --git a/pkg/searchapp/test.go b/pkg/searchapp/test.go deleted file mode 100644 index e33e2f7..0000000 --- a/pkg/searchapp/test.go +++ /dev/null @@ -1,23 +0,0 @@ -package searchapp - -import ( - "math" - - "github.com/curusarn/resh/pkg/histcli" - "github.com/curusarn/resh/pkg/msg" - "github.com/curusarn/resh/pkg/records" -) - -// LoadHistoryFromFile ... -func LoadHistoryFromFile(historyPath string, numLines int) msg.CliResponse { - recs := records.LoadFromFile(historyPath, math.MaxInt32) - if numLines != 0 && numLines < len(recs) { - recs = recs[:numLines] - } - cliRecords := histcli.New() - for i := len(recs) - 1; i >= 0; i-- { - rec := recs[i] - cliRecords.AddRecord(rec) - } - return msg.CliResponse{CliRecords: cliRecords.List} -} diff --git a/pkg/sesshist/sesshist.go b/pkg/sesshist/sesshist.go deleted file mode 100644 index f722764..0000000 --- a/pkg/sesshist/sesshist.go +++ /dev/null @@ -1,243 +0,0 @@ -package sesshist - -import ( - "errors" - "log" - "strconv" - "strings" - "sync" - - "github.com/curusarn/resh/pkg/histfile" - "github.com/curusarn/resh/pkg/histlist" - "github.com/curusarn/resh/pkg/records" -) - -// Dispatch Recall() calls to an apropriate session history (sesshist) -type Dispatch struct { - sessions map[string]*sesshist - mutex sync.RWMutex - - history *histfile.Histfile - historyInitSize int -} - -// NewDispatch creates a new sesshist.Dispatch and starts necessary gorutines -func NewDispatch(sessionsToInit chan records.Record, sessionsToDrop chan string, - recordsToAdd chan records.Record, history *histfile.Histfile, historyInitSize int) *Dispatch { - - s := Dispatch{ - sessions: map[string]*sesshist{}, - history: history, - historyInitSize: historyInitSize, - } - go s.sessionInitializer(sessionsToInit) - go s.sessionDropper(sessionsToDrop) - go s.recordAdder(recordsToAdd) - return &s -} - -func (s *Dispatch) sessionInitializer(sessionsToInit chan records.Record) { - for { - record := <-sessionsToInit - log.Println("sesshist: got session to init - " + record.SessionID) - s.initSession(record.SessionID, record.Shell) - } -} - -func (s *Dispatch) sessionDropper(sessionsToDrop chan string) { - for { - sessionID := <-sessionsToDrop - log.Println("sesshist: got session to drop - " + sessionID) - s.dropSession(sessionID) - } -} - -func (s *Dispatch) recordAdder(recordsToAdd chan records.Record) { - for { - record := <-recordsToAdd - if record.PartOne { - log.Println("sesshist: got record to add - " + record.CmdLine) - s.addRecentRecord(record.SessionID, record) - } else { - // this inits session on RESH update - s.checkSession(record.SessionID, record.Shell) - } - // TODO: we will need to handle part2 as well eventually - } -} - -func (s *Dispatch) checkSession(sessionID, shell string) { - s.mutex.RLock() - _, found := s.sessions[sessionID] - s.mutex.RUnlock() - if found == false { - err := s.initSession(sessionID, shell) - if err != nil { - log.Println("sesshist: Error while checking session:", err) - } - } -} - -// InitSession struct -func (s *Dispatch) initSession(sessionID, shell string) error { - log.Println("sesshist: initializing session - " + sessionID) - s.mutex.RLock() - _, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == true { - return errors.New("sesshist ERROR: Can't INIT already existing session " + sessionID) - } - - log.Println("sesshist: loading history to populate session - " + sessionID) - historyCmdLines := s.history.GetRecentCmdLines(shell, s.historyInitSize) - - s.mutex.Lock() - defer s.mutex.Unlock() - // init sesshist and populate it with history loaded from file - s.sessions[sessionID] = &sesshist{ - recentCmdLines: historyCmdLines, - } - log.Println("sesshist: session init done - " + sessionID) - return nil -} - -// DropSession struct -func (s *Dispatch) dropSession(sessionID string) error { - s.mutex.RLock() - _, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == false { - return errors.New("sesshist ERROR: Can't DROP not existing session " + sessionID) - } - - s.mutex.Lock() - defer s.mutex.Unlock() - delete(s.sessions, sessionID) - return nil -} - -// AddRecent record to session -func (s *Dispatch) addRecentRecord(sessionID string, record records.Record) error { - log.Println("sesshist: Adding a record, RLocking main lock ...") - s.mutex.RLock() - log.Println("sesshist: Getting a session ...") - session, found := s.sessions[sessionID] - log.Println("sesshist: RUnlocking main lock ...") - s.mutex.RUnlock() - - if found == false { - log.Println("sesshist ERROR: addRecentRecord(): No session history for SessionID " + sessionID + " - creating session history.") - s.initSession(sessionID, record.Shell) - return s.addRecentRecord(sessionID, record) - } - log.Println("sesshist: RLocking session lock (w/ defer) ...") - session.mutex.Lock() - defer session.mutex.Unlock() - session.recentRecords = append(session.recentRecords, record) - session.recentCmdLines.AddCmdLine(record.CmdLine) - log.Println("sesshist: record:", record.CmdLine, "; added to session:", sessionID, - "; session len:", len(session.recentCmdLines.List), "; session len (records):", len(session.recentRecords)) - return nil -} - -// Recall command from recent session history -func (s *Dispatch) Recall(sessionID string, histno int, prefix string) (string, error) { - log.Println("sesshist - recall: RLocking main lock ...") - s.mutex.RLock() - log.Println("sesshist - recall: Getting session history struct ...") - session, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == false { - // TODO: propagate actual shell here so we can use it - go s.initSession(sessionID, "bash") - return "", errors.New("sesshist ERROR: No session history for SessionID " + sessionID + " - creating one ...") - } - log.Println("sesshist - recall: Locking session lock ...") - session.mutex.Lock() - defer session.mutex.Unlock() - if prefix == "" { - log.Println("sesshist - recall: Getting records by histno ...") - return session.getRecordByHistno(histno) - } - log.Println("sesshist - recall: Searching for records by prefix ...") - return session.searchRecordByPrefix(prefix, histno) -} - -// Inspect commands in recent session history -func (s *Dispatch) Inspect(sessionID string, count int) ([]string, error) { - prefix := "" - log.Println("sesshist - inspect: RLocking main lock ...") - s.mutex.RLock() - log.Println("sesshist - inspect: Getting session history struct ...") - session, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == false { - // go s.initSession(sessionID) - return nil, errors.New("sesshist ERROR: No session history for SessionID " + sessionID + " - should we create one?") - } - log.Println("sesshist - inspect: Locking session lock ...") - session.mutex.Lock() - defer session.mutex.Unlock() - if prefix == "" { - log.Println("sesshist - inspect: Getting records by histno ...") - idx := len(session.recentCmdLines.List) - count - if idx < 0 { - idx = 0 - } - return session.recentCmdLines.List[idx:], nil - } - log.Println("sesshist - inspect: Searching for records by prefix ... ERROR - Not implemented") - return nil, errors.New("sesshist ERROR: Inspect - Searching for records by prefix Not implemented yet") -} - -type sesshist struct { - mutex sync.Mutex - recentRecords []records.Record - recentCmdLines histlist.Histlist -} - -func (s *sesshist) getRecordByHistno(histno int) (string, error) { - // addRecords() appends records to the end of the slice - // -> this func handles the indexing - if histno == 0 { - return "", errors.New("sesshist ERROR: 'histno == 0' is not a record from history") - } - if histno < 0 { - return "", errors.New("sesshist ERROR: 'histno < 0' is a command from future (not supperted yet)") - } - index := len(s.recentCmdLines.List) - histno - if index < 0 { - return "", errors.New("sesshist ERROR: 'histno > number of commands in the session' (" + strconv.Itoa(len(s.recentCmdLines.List)) + ")") - } - return s.recentCmdLines.List[index], nil -} - -func (s *sesshist) searchRecordByPrefix(prefix string, histno int) (string, error) { - if histno == 0 { - return "", errors.New("sesshist ERROR: 'histno == 0' is not a record from history") - } - if histno < 0 { - return "", errors.New("sesshist ERROR: 'histno < 0' is a command from future (not supperted yet)") - } - index := len(s.recentCmdLines.List) - histno - if index < 0 { - return "", errors.New("sesshist ERROR: 'histno > number of commands in the session' (" + strconv.Itoa(len(s.recentCmdLines.List)) + ")") - } - cmdLines := []string{} - for i := len(s.recentCmdLines.List) - 1; i >= 0; i-- { - if strings.HasPrefix(s.recentCmdLines.List[i], prefix) { - cmdLines = append(cmdLines, s.recentCmdLines.List[i]) - if len(cmdLines) >= histno { - break - } - } - } - if len(cmdLines) < histno { - return "", errors.New("sesshist ERROR: 'histno > number of commands matching with given prefix' (" + strconv.Itoa(len(cmdLines)) + ")") - } - return cmdLines[histno-1], nil -} diff --git a/pkg/sesswatch/sesswatch.go b/pkg/sesswatch/sesswatch.go deleted file mode 100644 index ae32bc4..0000000 --- a/pkg/sesswatch/sesswatch.go +++ /dev/null @@ -1,78 +0,0 @@ -package sesswatch - -import ( - "log" - "sync" - "time" - - "github.com/curusarn/resh/pkg/records" - "github.com/mitchellh/go-ps" -) - -type sesswatch struct { - sessionsToDrop []chan string - sleepSeconds uint - - watchedSessions map[string]bool - mutex sync.Mutex -} - -// Go runs the session watcher - watches sessions and sends -func Go(sessionsToWatch chan records.Record, sessionsToWatchRecords chan records.Record, sessionsToDrop []chan string, sleepSeconds uint) { - sw := sesswatch{sessionsToDrop: sessionsToDrop, sleepSeconds: sleepSeconds, watchedSessions: map[string]bool{}} - go sw.waiter(sessionsToWatch, sessionsToWatchRecords) -} - -func (s *sesswatch) waiter(sessionsToWatch chan records.Record, sessionsToWatchRecords chan records.Record) { - for { - func() { - select { - case record := <-sessionsToWatch: - // normal way to start watching a session - id := record.SessionID - pid := record.SessionPID - s.mutex.Lock() - defer s.mutex.Unlock() - if s.watchedSessions[id] == false { - log.Println("sesswatch: start watching NEW session ~ pid:", id, "~", pid) - s.watchedSessions[id] = true - go s.watcher(id, pid) - } - case record := <-sessionsToWatchRecords: - // additional safety - watch sessions that were never properly initialized - id := record.SessionID - pid := record.SessionPID - s.mutex.Lock() - defer s.mutex.Unlock() - if s.watchedSessions[id] == false { - log.Println("sesswatch WARN: start watching NEW session (based on /record) ~ pid:", id, "~", pid) - s.watchedSessions[id] = true - go s.watcher(id, pid) - } - } - }() - } -} - -func (s *sesswatch) watcher(sessionID string, sessionPID int) { - for { - time.Sleep(time.Duration(s.sleepSeconds) * time.Second) - proc, err := ps.FindProcess(sessionPID) - if err != nil { - log.Println("sesswatch ERROR: error while finding process:", sessionPID) - } else if proc == nil { - log.Println("sesswatch: Dropping session ~ pid:", sessionID, "~", sessionPID) - func() { - s.mutex.Lock() - defer s.mutex.Unlock() - s.watchedSessions[sessionID] = false - }() - for _, ch := range s.sessionsToDrop { - log.Println("sesswatch: sending 'drop session' message ...") - ch <- sessionID - log.Println("sesswatch: sending 'drop session' message DONE") - } - break - } - } -} diff --git a/pkg/signalhandler/signalhander.go b/pkg/signalhandler/signalhander.go deleted file mode 100644 index 5d2233e..0000000 --- a/pkg/signalhandler/signalhander.go +++ /dev/null @@ -1,65 +0,0 @@ -package signalhandler - -import ( - "context" - "log" - "net/http" - "os" - "os/signal" - "strconv" - "syscall" - "time" -) - -func sendSignals(sig os.Signal, subscribers []chan os.Signal, done chan string) { - for _, sub := range subscribers { - sub <- sig - } - chanCount := len(subscribers) - start := time.Now() - delay := time.Millisecond * 100 - timeout := time.Millisecond * 2000 - - for { - select { - case _ = <-done: - chanCount-- - if chanCount == 0 { - log.Println("signalhandler: All components shut down successfully") - return - } - default: - time.Sleep(delay) - } - if time.Since(start) > timeout { - log.Println("signalhandler: Timouted while waiting for proper shutdown - " + strconv.Itoa(chanCount) + " boxes are up after " + timeout.String()) - return - } - } -} - -// Run catches and handles signals -func Run(subscribers []chan os.Signal, done chan string, server *http.Server) { - signals := make(chan os.Signal, 1) - - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - var sig os.Signal - for { - sig := <-signals - log.Printf("signalhandler: Got signal '%s'\n", sig.String()) - if sig == syscall.SIGTERM { - // Shutdown daemon on SIGTERM - break - } - log.Printf("signalhandler: Ignoring signal '%s'. Send SIGTERM to trigger shutdown.\n", sig.String()) - } - - log.Println("signalhandler: Sending shutdown signals to components") - sendSignals(sig, subscribers, done) - - log.Println("signalhandler: Shutting down the server") - if err := server.Shutdown(context.Background()); err != nil { - log.Printf("HTTP server Shutdown: %v", err) - } -} diff --git a/pkg/strat/directory-sensitive.go b/pkg/strat/directory-sensitive.go deleted file mode 100644 index 89d030e..0000000 --- a/pkg/strat/directory-sensitive.go +++ /dev/null @@ -1,47 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// DirectorySensitive prediction/recommendation strategy -type DirectorySensitive struct { - history map[string][]string - lastPwd string -} - -// Init see name -func (s *DirectorySensitive) Init() { - s.history = map[string][]string{} -} - -// GetTitleAndDescription see name -func (s *DirectorySensitive) GetTitleAndDescription() (string, string) { - return "directory sensitive (recent)", "Use recent commands executed is the same directory" -} - -// GetCandidates see name -func (s *DirectorySensitive) GetCandidates() []string { - return s.history[s.lastPwd] -} - -// AddHistoryRecord see name -func (s *DirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error { - // work on history for PWD - pwd := record.Pwd - // remove previous occurance of record - for i, cmd := range s.history[pwd] { - if cmd == record.CmdLine { - s.history[pwd] = append(s.history[pwd][:i], s.history[pwd][i+1:]...) - } - } - // append new record - s.history[pwd] = append([]string{record.CmdLine}, s.history[pwd]...) - s.lastPwd = record.PwdAfter - return nil -} - -// ResetHistory see name -func (s *DirectorySensitive) ResetHistory() error { - s.Init() - s.history = map[string][]string{} - return nil -} diff --git a/pkg/strat/dummy.go b/pkg/strat/dummy.go deleted file mode 100644 index fc813f2..0000000 --- a/pkg/strat/dummy.go +++ /dev/null @@ -1,29 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// Dummy prediction/recommendation strategy -type Dummy struct { - history []string -} - -// GetTitleAndDescription see name -func (s *Dummy) GetTitleAndDescription() (string, string) { - return "dummy", "Return empty candidate list" -} - -// GetCandidates see name -func (s *Dummy) GetCandidates() []string { - return nil -} - -// AddHistoryRecord see name -func (s *Dummy) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append(s.history, record.CmdLine) - return nil -} - -// ResetHistory see name -func (s *Dummy) ResetHistory() error { - return nil -} diff --git a/pkg/strat/dynamic-record-distance.go b/pkg/strat/dynamic-record-distance.go deleted file mode 100644 index 1f779c2..0000000 --- a/pkg/strat/dynamic-record-distance.go +++ /dev/null @@ -1,91 +0,0 @@ -package strat - -import ( - "math" - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" -) - -// DynamicRecordDistance prediction/recommendation strategy -type DynamicRecordDistance struct { - history []records.EnrichedRecord - DistParams records.DistParams - pwdHistogram map[string]int - realPwdHistogram map[string]int - gitOriginHistogram map[string]int - MaxDepth int - Label string -} - -type strDynDistEntry struct { - cmdLine string - distance float64 -} - -// Init see name -func (s *DynamicRecordDistance) Init() { - s.history = nil - s.pwdHistogram = map[string]int{} - s.realPwdHistogram = map[string]int{} - s.gitOriginHistogram = map[string]int{} -} - -// GetTitleAndDescription see name -func (s *DynamicRecordDistance) GetTitleAndDescription() (string, string) { - return "dynamic record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use TF-IDF record distance to recommend commands" -} - -func (s *DynamicRecordDistance) idf(count int) float64 { - return math.Log(float64(len(s.history)) / float64(count)) -} - -// GetCandidates see name -func (s *DynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { - if len(s.history) == 0 { - return nil - } - var mapItems []strDynDistEntry - for i, record := range s.history { - if s.MaxDepth != 0 && i > s.MaxDepth { - break - } - distParams := records.DistParams{ - Pwd: s.DistParams.Pwd * s.idf(s.pwdHistogram[strippedRecord.PwdAfter]), - RealPwd: s.DistParams.RealPwd * s.idf(s.realPwdHistogram[strippedRecord.RealPwdAfter]), - Git: s.DistParams.Git * s.idf(s.gitOriginHistogram[strippedRecord.GitOriginRemote]), - Time: s.DistParams.Time, - SessionID: s.DistParams.SessionID, - } - distance := record.DistanceTo(strippedRecord, distParams) - mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance}) - } - sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) - var hist []string - histSet := map[string]bool{} - for _, item := range mapItems { - if histSet[item.cmdLine] { - continue - } - histSet[item.cmdLine] = true - hist = append(hist, item.cmdLine) - } - return hist -} - -// AddHistoryRecord see name -func (s *DynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { - // append record to front - s.history = append([]records.EnrichedRecord{*record}, s.history...) - s.pwdHistogram[record.Pwd]++ - s.realPwdHistogram[record.RealPwd]++ - s.gitOriginHistogram[record.GitOriginRemote]++ - return nil -} - -// ResetHistory see name -func (s *DynamicRecordDistance) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/frequent.go b/pkg/strat/frequent.go deleted file mode 100644 index ff3b912..0000000 --- a/pkg/strat/frequent.go +++ /dev/null @@ -1,53 +0,0 @@ -package strat - -import ( - "sort" - - "github.com/curusarn/resh/pkg/records" -) - -// Frequent prediction/recommendation strategy -type Frequent struct { - history map[string]int -} - -type strFrqEntry struct { - cmdLine string - count int -} - -// Init see name -func (s *Frequent) Init() { - s.history = map[string]int{} -} - -// GetTitleAndDescription see name -func (s *Frequent) GetTitleAndDescription() (string, string) { - return "frequent", "Use frequent commands" -} - -// GetCandidates see name -func (s *Frequent) GetCandidates() []string { - var mapItems []strFrqEntry - for cmdLine, count := range s.history { - mapItems = append(mapItems, strFrqEntry{cmdLine, count}) - } - sort.Slice(mapItems, func(i int, j int) bool { return mapItems[i].count > mapItems[j].count }) - var hist []string - for _, item := range mapItems { - hist = append(hist, item.cmdLine) - } - return hist -} - -// AddHistoryRecord see name -func (s *Frequent) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history[record.CmdLine]++ - return nil -} - -// ResetHistory see name -func (s *Frequent) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/markov-chain-cmd.go b/pkg/strat/markov-chain-cmd.go deleted file mode 100644 index b1fa2f5..0000000 --- a/pkg/strat/markov-chain-cmd.go +++ /dev/null @@ -1,97 +0,0 @@ -package strat - -import ( - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" - "github.com/mb-14/gomarkov" -) - -// MarkovChainCmd prediction/recommendation strategy -type MarkovChainCmd struct { - Order int - history []strMarkCmdHistoryEntry - historyCmds []string -} - -type strMarkCmdHistoryEntry struct { - cmd string - cmdLine string -} - -type strMarkCmdEntry struct { - cmd string - transProb float64 -} - -// Init see name -func (s *MarkovChainCmd) Init() { - s.history = nil - s.historyCmds = nil -} - -// GetTitleAndDescription see name -func (s *MarkovChainCmd) GetTitleAndDescription() (string, string) { - return "command-based markov chain (order " + strconv.Itoa(s.Order) + ")", "Use command-based markov chain to recommend commands" -} - -// GetCandidates see name -func (s *MarkovChainCmd) GetCandidates() []string { - if len(s.history) < s.Order { - var hist []string - for _, item := range s.history { - hist = append(hist, item.cmdLine) - } - return hist - } - chain := gomarkov.NewChain(s.Order) - - chain.Add(s.historyCmds) - - cmdsSet := map[string]bool{} - var entries []strMarkCmdEntry - for _, cmd := range s.historyCmds { - if cmdsSet[cmd] { - continue - } - cmdsSet[cmd] = true - prob, _ := chain.TransitionProbability(cmd, s.historyCmds[len(s.historyCmds)-s.Order:]) - entries = append(entries, strMarkCmdEntry{cmd: cmd, transProb: prob}) - } - sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) - var hist []string - histSet := map[string]bool{} - for i := len(s.history) - 1; i >= 0; i-- { - if histSet[s.history[i].cmdLine] { - continue - } - histSet[s.history[i].cmdLine] = true - if s.history[i].cmd == entries[0].cmd { - hist = append(hist, s.history[i].cmdLine) - } - } - // log.Println("################") - // log.Println(s.history[len(s.history)-s.order:]) - // log.Println(" -> ") - // x := math.Min(float64(len(hist)), 3) - // log.Println(entries[:int(x)]) - // x = math.Min(float64(len(hist)), 5) - // log.Println(hist[:int(x)]) - // log.Println("################") - return hist -} - -// AddHistoryRecord see name -func (s *MarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command}) - s.historyCmds = append(s.historyCmds, record.Command) - // s.historySet[record.CmdLine] = true - return nil -} - -// ResetHistory see name -func (s *MarkovChainCmd) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/markov-chain.go b/pkg/strat/markov-chain.go deleted file mode 100644 index 50c7fdc..0000000 --- a/pkg/strat/markov-chain.go +++ /dev/null @@ -1,76 +0,0 @@ -package strat - -import ( - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" - "github.com/mb-14/gomarkov" -) - -// MarkovChain prediction/recommendation strategy -type MarkovChain struct { - Order int - history []string -} - -type strMarkEntry struct { - cmdLine string - transProb float64 -} - -// Init see name -func (s *MarkovChain) Init() { - s.history = nil -} - -// GetTitleAndDescription see name -func (s *MarkovChain) GetTitleAndDescription() (string, string) { - return "markov chain (order " + strconv.Itoa(s.Order) + ")", "Use markov chain to recommend commands" -} - -// GetCandidates see name -func (s *MarkovChain) GetCandidates() []string { - if len(s.history) < s.Order { - return s.history - } - chain := gomarkov.NewChain(s.Order) - - chain.Add(s.history) - - cmdLinesSet := map[string]bool{} - var entries []strMarkEntry - for _, cmdLine := range s.history { - if cmdLinesSet[cmdLine] { - continue - } - cmdLinesSet[cmdLine] = true - prob, _ := chain.TransitionProbability(cmdLine, s.history[len(s.history)-s.Order:]) - entries = append(entries, strMarkEntry{cmdLine: cmdLine, transProb: prob}) - } - sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) - var hist []string - for _, item := range entries { - hist = append(hist, item.cmdLine) - } - // log.Println("################") - // log.Println(s.history[len(s.history)-s.order:]) - // log.Println(" -> ") - // x := math.Min(float64(len(hist)), 5) - // log.Println(hist[:int(x)]) - // log.Println("################") - return hist -} - -// AddHistoryRecord see name -func (s *MarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append(s.history, record.CmdLine) - // s.historySet[record.CmdLine] = true - return nil -} - -// ResetHistory see name -func (s *MarkovChain) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/random.go b/pkg/strat/random.go deleted file mode 100644 index 0ff52f1..0000000 --- a/pkg/strat/random.go +++ /dev/null @@ -1,57 +0,0 @@ -package strat - -import ( - "math/rand" - "time" - - "github.com/curusarn/resh/pkg/records" -) - -// Random prediction/recommendation strategy -type Random struct { - CandidatesSize int - history []string - historySet map[string]bool -} - -// Init see name -func (s *Random) Init() { - s.history = nil - s.historySet = map[string]bool{} -} - -// GetTitleAndDescription see name -func (s *Random) GetTitleAndDescription() (string, string) { - return "random", "Use random commands" -} - -// GetCandidates see name -func (s *Random) GetCandidates() []string { - seed := time.Now().UnixNano() - rand.Seed(seed) - var candidates []string - candidateSet := map[string]bool{} - for len(candidates) < s.CandidatesSize && len(candidates)*2 < len(s.historySet) { - x := rand.Intn(len(s.history)) - candidate := s.history[x] - if candidateSet[candidate] == false { - candidateSet[candidate] = true - candidates = append(candidates, candidate) - continue - } - } - return candidates -} - -// AddHistoryRecord see name -func (s *Random) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append([]string{record.CmdLine}, s.history...) - s.historySet[record.CmdLine] = true - return nil -} - -// ResetHistory see name -func (s *Random) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/recent-bash.go b/pkg/strat/recent-bash.go deleted file mode 100644 index ace3571..0000000 --- a/pkg/strat/recent-bash.go +++ /dev/null @@ -1,56 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// RecentBash prediction/recommendation strategy -type RecentBash struct { - histfile []string - histfileSnapshot map[string][]string - history map[string][]string -} - -// Init see name -func (s *RecentBash) Init() { - s.histfileSnapshot = map[string][]string{} - s.history = map[string][]string{} -} - -// GetTitleAndDescription see name -func (s *RecentBash) GetTitleAndDescription() (string, string) { - return "recent (bash-like)", "Behave like bash" -} - -// GetCandidates see name -func (s *RecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string { - // populate the local history from histfile - if s.histfileSnapshot[strippedRecord.SessionID] == nil { - s.histfileSnapshot[strippedRecord.SessionID] = s.histfile - } - return append(s.history[strippedRecord.SessionID], s.histfileSnapshot[strippedRecord.SessionID]...) -} - -// AddHistoryRecord see name -func (s *RecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { - // remove previous occurance of record - for i, cmd := range s.history[record.SessionID] { - if cmd == record.CmdLine { - s.history[record.SessionID] = append(s.history[record.SessionID][:i], s.history[record.SessionID][i+1:]...) - } - } - // append new record - s.history[record.SessionID] = append([]string{record.CmdLine}, s.history[record.SessionID]...) - - if record.LastRecordOfSession { - // append history of the session to histfile and clear session history - s.histfile = append(s.history[record.SessionID], s.histfile...) - s.histfileSnapshot[record.SessionID] = nil - s.history[record.SessionID] = nil - } - return nil -} - -// ResetHistory see name -func (s *RecentBash) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/recent.go b/pkg/strat/recent.go deleted file mode 100644 index 157b52c..0000000 --- a/pkg/strat/recent.go +++ /dev/null @@ -1,37 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// Recent prediction/recommendation strategy -type Recent struct { - history []string -} - -// GetTitleAndDescription see name -func (s *Recent) GetTitleAndDescription() (string, string) { - return "recent", "Use recent commands" -} - -// GetCandidates see name -func (s *Recent) GetCandidates() []string { - return s.history -} - -// AddHistoryRecord see name -func (s *Recent) AddHistoryRecord(record *records.EnrichedRecord) error { - // remove previous occurance of record - for i, cmd := range s.history { - if cmd == record.CmdLine { - s.history = append(s.history[:i], s.history[i+1:]...) - } - } - // append new record - s.history = append([]string{record.CmdLine}, s.history...) - return nil -} - -// ResetHistory see name -func (s *Recent) ResetHistory() error { - s.history = nil - return nil -} diff --git a/pkg/strat/record-distance.go b/pkg/strat/record-distance.go deleted file mode 100644 index e582584..0000000 --- a/pkg/strat/record-distance.go +++ /dev/null @@ -1,70 +0,0 @@ -package strat - -import ( - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" -) - -// RecordDistance prediction/recommendation strategy -type RecordDistance struct { - history []records.EnrichedRecord - DistParams records.DistParams - MaxDepth int - Label string -} - -type strDistEntry struct { - cmdLine string - distance float64 -} - -// Init see name -func (s *RecordDistance) Init() { - s.history = nil -} - -// GetTitleAndDescription see name -func (s *RecordDistance) GetTitleAndDescription() (string, string) { - return "record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use record distance to recommend commands" -} - -// GetCandidates see name -func (s *RecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { - if len(s.history) == 0 { - return nil - } - var mapItems []strDistEntry - for i, record := range s.history { - if s.MaxDepth != 0 && i > s.MaxDepth { - break - } - distance := record.DistanceTo(strippedRecord, s.DistParams) - mapItems = append(mapItems, strDistEntry{record.CmdLine, distance}) - } - sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) - var hist []string - histSet := map[string]bool{} - for _, item := range mapItems { - if histSet[item.cmdLine] { - continue - } - histSet[item.cmdLine] = true - hist = append(hist, item.cmdLine) - } - return hist -} - -// AddHistoryRecord see name -func (s *RecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { - // append record to front - s.history = append([]records.EnrichedRecord{*record}, s.history...) - return nil -} - -// ResetHistory see name -func (s *RecordDistance) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/strat.go b/pkg/strat/strat.go deleted file mode 100644 index 28ac015..0000000 --- a/pkg/strat/strat.go +++ /dev/null @@ -1,46 +0,0 @@ -package strat - -import ( - "github.com/curusarn/resh/pkg/records" -) - -// ISimpleStrategy interface -type ISimpleStrategy interface { - GetTitleAndDescription() (string, string) - GetCandidates() []string - AddHistoryRecord(record *records.EnrichedRecord) error - ResetHistory() error -} - -// IStrategy interface -type IStrategy interface { - GetTitleAndDescription() (string, string) - GetCandidates(r records.EnrichedRecord) []string - AddHistoryRecord(record *records.EnrichedRecord) error - ResetHistory() error -} - -type simpleStrategyWrapper struct { - strategy ISimpleStrategy -} - -// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy -func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper { - return &simpleStrategyWrapper{strategy: strategy} -} - -func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) { - return s.strategy.GetTitleAndDescription() -} - -func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string { - return s.strategy.GetCandidates() -} - -func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error { - return s.strategy.AddHistoryRecord(r) -} - -func (s *simpleStrategyWrapper) ResetHistory() error { - return s.strategy.ResetHistory() -} diff --git a/record/legacy.go b/record/legacy.go new file mode 100644 index 0000000..3b913fb --- /dev/null +++ b/record/legacy.go @@ -0,0 +1,88 @@ +package record + +type Legacy struct { + // core + CmdLine string `json:"cmdLine"` + ExitCode int `json:"exitCode"` + Shell string `json:"shell"` + Uname string `json:"uname"` + SessionID string `json:"sessionId"` + RecordID string `json:"recordId"` + + // posix + Home string `json:"home"` + Lang string `json:"lang"` + LcAll string `json:"lcAll"` + Login string `json:"login"` + Pwd string `json:"pwd"` + PwdAfter string `json:"pwdAfter"` + ShellEnv string `json:"shellEnv"` + Term string `json:"term"` + + // non-posix"` + RealPwd string `json:"realPwd"` + RealPwdAfter string `json:"realPwdAfter"` + Pid int `json:"pid"` + SessionPID int `json:"sessionPid"` + Host string `json:"host"` + Hosttype string `json:"hosttype"` + Ostype string `json:"ostype"` + Machtype string `json:"machtype"` + Shlvl int `json:"shlvl"` + + // before after + TimezoneBefore string `json:"timezoneBefore"` + TimezoneAfter string `json:"timezoneAfter"` + + RealtimeBefore float64 `json:"realtimeBefore"` + RealtimeAfter float64 `json:"realtimeAfter"` + RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` + RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` + + RealtimeDuration float64 `json:"realtimeDuration"` + RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` + RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` + + GitDir string `json:"gitDir"` + GitRealDir string `json:"gitRealDir"` + GitOriginRemote string `json:"gitOriginRemote"` + GitDirAfter string `json:"gitDirAfter"` + GitRealDirAfter string `json:"gitRealDirAfter"` + GitOriginRemoteAfter string `json:"gitOriginRemoteAfter"` + MachineID string `json:"machineId"` + + OsReleaseID string `json:"osReleaseId"` + OsReleaseVersionID string `json:"osReleaseVersionId"` + OsReleaseIDLike string `json:"osReleaseIdLike"` + OsReleaseName string `json:"osReleaseName"` + OsReleasePrettyName string `json:"osReleasePrettyName"` + + ReshUUID string `json:"reshUuid"` + ReshVersion string `json:"reshVersion"` + ReshRevision string `json:"reshRevision"` + + // records come in two parts (collect and postcollect) + PartOne bool `json:"partOne,omitempty"` // false => part two + PartsMerged bool `json:"partsMerged"` + // special flag -> not an actual record but an session end + SessionExit bool `json:"sessionExit,omitempty"` + + // recall metadata + Recalled bool `json:"recalled"` + RecallHistno int `json:"recallHistno,omitempty"` + RecallStrategy string `json:"recallStrategy,omitempty"` + RecallActionsRaw string `json:"recallActionsRaw,omitempty"` + RecallActions []string `json:"recallActions,omitempty"` + RecallLastCmdLine string `json:"recallLastCmdLine"` + + // recall command + RecallPrefix string `json:"recallPrefix,omitempty"` + + // added by sanitizatizer + Sanitized bool `json:"sanitized,omitempty"` + CmdLength int `json:"cmdLength,omitempty"` + + // fields that are string here and int in older resh verisons + Cols interface{} `json:"cols"` + Lines interface{} `json:"lines"` +} diff --git a/record/record.go b/record/record.go new file mode 100644 index 0000000..07ad798 --- /dev/null +++ b/record/record.go @@ -0,0 +1,2 @@ +// Package record provides record types that are used in resh history files +package record diff --git a/record/v1.go b/record/v1.go new file mode 100644 index 0000000..6585051 --- /dev/null +++ b/record/v1.go @@ -0,0 +1,64 @@ +package record + +type V1 struct { + // flags + // deleted, favorite + // FIXME: is this the best way? .. what about string, separate fields, or something similar + Flags int `json:"flags"` + + // cmdline, exitcode + CmdLine string `json:"cmdLine"` + ExitCode int `json:"exitCode"` + + DeviceID string `json:"deviceID"` + SessionID string `json:"sessionID"` + // can we have a shorter uuid for record + RecordID string `json:"recordID"` + + // paths + // TODO: Do we need both pwd and real pwd? + Home string `json:"home"` + Pwd string `json:"pwd"` + RealPwd string `json:"realPwd"` + + // hostname + logname (not sure if we actually need logname) + // Logname string `json:"logname"` + // Device is usually hostname but not stricly hostname + // It can be configured in RESH configuration + Device string `json:"device"` + + // git info + // origin is the most important + GitOriginRemote string `json:"gitOriginRemote"` + // TODO: add GitBranch (v2 ?) + // maybe branch could be useful - e.g. in monorepo ?? + // GitBranch string `json:"gitBranch"` + + // what is this for ?? + // session watching needs this + // but I'm not sure if we need to save it + // records belong to sessions + // PID int `json:"pid"` + // needed for tracking of sessions but I think it shouldn't be part of V1 + // SessionPID int `json:"sessionPID"` + + // needed to because records are merged with parts with same "SessionID + Shlvl" + // I don't think we need to save it + // Shlvl int `json:"shlvl"` + + // time (before), duration of command + // time and duration are strings because we don't want unnecessary precision when they get serialized into json + // we could implement custom (un)marshalling but I don't see downsides of directly representing the values as strings + Time string `json:"time"` + Duration string `json:"duration"` + + // these look like internal stuff + + // records come in two parts (collect and postcollect) + PartOne bool `json:"partOne,omitempty"` // false => part two + PartsNotMerged bool `json:"partsNotMerged,omitempty"` + + // special flag -> not an actual record but an session end + // TODO: this shouldn't be part of serializable V1 record + SessionExit bool `json:"sessionExit,omitempty"` +} diff --git a/roadmap.md b/roadmap.md index efcc2d5..e2be272 100644 --- a/roadmap.md +++ b/roadmap.md @@ -41,5 +41,3 @@ - :heavy_check_mark: Linux - :white_check_mark: MacOS *(requires coreutils - `brew install coreutils`)* -- :heavy_check_mark: Provide a tool to sanitize the recorded history - diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 02e74c3..b331ec0 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -1,14 +1,6 @@ +#!/hint/sh __resh_reset_variables() { - __RESH_HISTNO=0 - __RESH_HISTNO_MAX="" - __RESH_HISTNO_ZERO_LINE="" - __RESH_HIST_PREV_LINE="" - __RESH_HIST_PREV_CURSOR="" # deprecated - __RESH_HIST_PREV_PREFIX="" - __RESH_HIST_RECALL_ACTIONS="" - __RESH_HIST_NO_PREFIX_MODE=0 - __RESH_HIST_RECALL_STRATEGY="" __RESH_RECORD_ID=$(__resh_get_uuid) } @@ -16,49 +8,18 @@ __resh_preexec() { # core __RESH_COLLECT=1 __RESH_CMDLINE="$1" # not local to preserve it for postcollect (useful as sanity check) - local fpath_last_run="$__RESH_XDG_CACHE_HOME/collect_last_run_out.txt" - __resh_collect --cmdLine "$__RESH_CMDLINE" \ - --recall-actions "$__RESH_HIST_RECALL_ACTIONS" \ - --recall-strategy "$__RESH_HIST_RECALL_STRATEGY" \ - --recall-last-cmdline "$__RESH_HIST_PREV_LINE" \ - >| "$fpath_last_run" 2>&1 || echo "resh-collect ERROR: $(head -n 1 $fpath_last_run)" + __resh_collect --cmdLine "$__RESH_CMDLINE" } # used for collect and collect --recall __resh_collect() { # posix - local __RESH_COLS="$COLUMNS" - local __RESH_LANG="$LANG" - local __RESH_LC_ALL="$LC_ALL" - # other LC ? - local __RESH_LINES="$LINES" - # __RESH_PATH="$PATH" local __RESH_PWD="$PWD" # non-posix local __RESH_SHLVL="$SHLVL" - local __RESH_GIT_CDUP; __RESH_GIT_CDUP="$(git rev-parse --show-cdup 2>/dev/null)" - local __RESH_GIT_CDUP_EXIT_CODE=$? local __RESH_GIT_REMOTE; __RESH_GIT_REMOTE="$(git remote get-url origin 2>/dev/null)" - local __RESH_GIT_REMOTE_EXIT_CODE=$? - #__RESH_GIT_TOPLEVEL="$(git rev-parse --show-toplevel)" - #__RESH_GIT_TOPLEVEL_EXIT_CODE=$? - if [ -n "${ZSH_VERSION-}" ]; then - # assume Zsh - local __RESH_PID="$$" # current pid - elif [ -n "${BASH_VERSION-}" ]; then - # assume Bash - if [ "${BASH_VERSINFO[0]}" -ge "4" ]; then - # $BASHPID is only available in bash4+ - # $$ is fairly similar so it should not be an issue - local __RESH_PID="$BASHPID" # current pid - else - local __RESH_PID="$$" # current pid - fi - fi - # time - local __RESH_TZ_BEFORE; __RESH_TZ_BEFORE=$(date +%z) # __RESH_RT_BEFORE="$EPOCHREALTIME" __RESH_RT_BEFORE=$(__resh_get_epochrealtime) @@ -72,48 +33,26 @@ __resh_collect() { fi elif [ "$__RESH_REVISION" != "$(resh-collect -revision)" ]; then # shellcheck source=shellrc.sh - source ~/.resh/shellrc + source ~/.resh/shellrc if [ "$__RESH_REVISION" != "$(resh-collect -revision)" ]; then echo "RESH WARNING: You probably just updated RESH - PLEASE RESTART OR RELOAD THIS TERMINAL SESSION (resh revision: $(resh-collect -revision); resh revision of this terminal session: ${__RESH_REVISION})" fi fi + # TODO: change how resh-uuid is read if [ "$__RESH_VERSION" = "$(resh-collect -version)" ] && [ "$__RESH_REVISION" = "$(resh-collect -revision)" ]; then resh-collect -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ -shell "$__RESH_SHELL" \ - -uname "$__RESH_UNAME" \ - -sessionId "$__RESH_SESSION_ID" \ - -recordId "$__RESH_RECORD_ID" \ - -cols "$__RESH_COLS" \ + -device "$__RESH_HOST" \ + -deviceID "$(cat ~/.resh/resh-uuid 2>/dev/null)" \ + -sessionID "$__RESH_SESSION_ID" \ + -recordID "$__RESH_RECORD_ID" \ -home "$__RESH_HOME" \ - -lang "$__RESH_LANG" \ - -lcAll "$__RESH_LC_ALL" \ - -lines "$__RESH_LINES" \ - -login "$__RESH_LOGIN" \ -pwd "$__RESH_PWD" \ - -shellEnv "$__RESH_SHELL_ENV" \ - -term "$__RESH_TERM" \ - -pid "$__RESH_PID" \ - -sessionPid "$__RESH_SESSION_PID" \ - -host "$__RESH_HOST" \ - -hosttype "$__RESH_HOSTTYPE" \ - -ostype "$__RESH_OSTYPE" \ - -machtype "$__RESH_MACHTYPE" \ + -sessionPID "$__RESH_SESSION_PID" \ -shlvl "$__RESH_SHLVL" \ - -gitCdup "$__RESH_GIT_CDUP" \ - -gitCdupExitCode "$__RESH_GIT_CDUP_EXIT_CODE" \ -gitRemote "$__RESH_GIT_REMOTE" \ - -gitRemoteExitCode "$__RESH_GIT_REMOTE_EXIT_CODE" \ - -realtimeBefore "$__RESH_RT_BEFORE" \ - -realtimeSession "$__RESH_RT_SESSION" \ - -realtimeSessSinceBoot "$__RESH_RT_SESS_SINCE_BOOT" \ - -timezoneBefore "$__RESH_TZ_BEFORE" \ - -osReleaseId "$__RESH_OS_RELEASE_ID" \ - -osReleaseVersionId "$__RESH_OS_RELEASE_VERSION_ID" \ - -osReleaseIdLike "$__RESH_OS_RELEASE_ID_LIKE" \ - -osReleaseName "$__RESH_OS_RELEASE_NAME" \ - -osReleasePrettyName "$__RESH_OS_RELEASE_PRETTY_NAME" \ - -histno "$__RESH_HISTNO" \ + -time "$__RESH_RT_BEFORE" \ "$@" return $? fi @@ -123,20 +62,8 @@ __resh_collect() { __resh_precmd() { local __RESH_EXIT_CODE=$? local __RESH_RT_AFTER - local __RESH_TZ_AFTER - local __RESH_PWD_AFTER - local __RESH_GIT_CDUP_AFTER - local __RESH_GIT_CDUP_EXIT_CODE_AFTER - local __RESH_GIT_REMOTE_AFTER - local __RESH_GIT_REMOTE_EXIT_CODE_AFTER local __RESH_SHLVL="$SHLVL" __RESH_RT_AFTER=$(__resh_get_epochrealtime) - __RESH_TZ_AFTER=$(date +%z) - __RESH_PWD_AFTER="$PWD" - __RESH_GIT_CDUP_AFTER="$(git rev-parse --show-cdup 2>/dev/null)" - __RESH_GIT_CDUP_EXIT_CODE_AFTER=$? - __RESH_GIT_REMOTE_AFTER="$(git remote get-url origin 2>/dev/null)" - __RESH_GIT_REMOTE_EXIT_CODE_AFTER=$? if [ -n "${__RESH_COLLECT}" ]; then if [ "$__RESH_VERSION" != "$(resh-postcollect -version)" ]; then # shellcheck source=shellrc.sh @@ -154,24 +81,14 @@ __resh_precmd() { fi fi if [ "$__RESH_VERSION" = "$(resh-postcollect -version)" ] && [ "$__RESH_REVISION" = "$(resh-postcollect -revision)" ]; then - local fpath_last_run="$__RESH_XDG_CACHE_HOME/postcollect_last_run_out.txt" resh-postcollect -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ - -cmdLine "$__RESH_CMDLINE" \ - -realtimeBefore "$__RESH_RT_BEFORE" \ + -timeBefore "$__RESH_RT_BEFORE" \ -exitCode "$__RESH_EXIT_CODE" \ - -sessionId "$__RESH_SESSION_ID" \ - -recordId "$__RESH_RECORD_ID" \ - -shell "$__RESH_SHELL" \ + -sessionID "$__RESH_SESSION_ID" \ + -recordID "$__RESH_RECORD_ID" \ -shlvl "$__RESH_SHLVL" \ - -pwdAfter "$__RESH_PWD_AFTER" \ - -gitCdupAfter "$__RESH_GIT_CDUP_AFTER" \ - -gitCdupExitCodeAfter "$__RESH_GIT_CDUP_EXIT_CODE_AFTER" \ - -gitRemoteAfter "$__RESH_GIT_REMOTE_AFTER" \ - -gitRemoteExitCodeAfter "$__RESH_GIT_REMOTE_EXIT_CODE_AFTER" \ - -realtimeAfter "$__RESH_RT_AFTER" \ - -timezoneAfter "$__RESH_TZ_AFTER" \ - >| "$fpath_last_run" 2>&1 || echo "resh-postcollect ERROR: $(head -n 1 $fpath_last_run)" + -timeAfter "$__RESH_RT_AFTER" fi __resh_reset_variables fi diff --git a/scripts/install.sh b/scripts/install.sh index 1b22d15..e5fcb6e 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -31,7 +31,7 @@ if [ "$bash_too_old" = true ]; then if [ "$login_shell" = bash ]; then echo " > Your bash version is old." echo " > Bash is also your login shell." - echo " > Updating to bash 4.3+ is strongly RECOMMENDED!" + echo " > Updating to bash 4.3+ is STRONGLY RECOMMENDED!" else echo " > Your bash version is old" echo " > Bash is not your login shell so it should not be an issue." @@ -52,7 +52,7 @@ else if [ "$login_shell" = zsh ]; then echo " > Your zsh version is old." echo " > Zsh is also your login shell." - echo " > Updating to Zsh 5.0+ is strongly RECOMMENDED!" + echo " > Updating to Zsh 5.0+ is STRONGLY RECOMMENDED!" else echo " > Your zsh version is old" echo " > Zsh is not your login shell so it should not be an issue." @@ -93,7 +93,22 @@ fi # # shellcheck disable=2034 # read -r x -echo +echo "Backing up previous installation" +#./bin/resh-install-utils backup +# TODO: ~/.resh -> XDG_DATA_HOME/resh/rollback/ +# TODO: ~/XDG_DATA_HOME/resh/history.reshjson -> XDG_DATA/resh/rollback/ +# TODO: what about legacy history locations +# TODO: ~/XDG_DATA_HOME/resh/log.json -> XDG_DATA/resh/rollback/ + +echo "Cleaning up installation directory ..." +rm ~/.resh/bin/* 2>/dev/null ||: +rm ~/.resh/* 2>/dev/null ||: +# TODO: put this behind version condition +# backward compatibility: We have a new location for resh history file +[ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json + +#[ ! -f ~/.resh_history.json ] || mv ~/.resh_history.json $XDG .resh_history.json + echo "Creating directories ..." mkdir_if_not_exists() { @@ -104,8 +119,6 @@ mkdir_if_not_exists() { mkdir_if_not_exists ~/.resh mkdir_if_not_exists ~/.resh/bin -mkdir_if_not_exists ~/.resh/bash_completion.d -mkdir_if_not_exists ~/.resh/zsh_completion.d mkdir_if_not_exists ~/.config echo "Copying files ..." @@ -116,41 +129,12 @@ cp -f scripts/shellrc.sh ~/.resh/shellrc cp -f scripts/reshctl.sh scripts/widgets.sh scripts/hooks.sh scripts/util.sh ~/.resh/ cp -f scripts/rawinstall.sh ~/.resh/ -update_config() { - version=$1 - key=$2 - value=$3 - # TODO: create bin/semver-lt - if bin/semver-lt "${__RESH_VERSION:-0.0.0}" "$1" && [ "$(bin/resh-config -key $key)" != "$value" ] ; then - echo " * config option $key was updated to $value" - # TODO: enable resh-config value setting - # resh-config -key "$key" -value "$value" - fi -} - - -# Do not overwrite config if it exists -if [ ! -f ~/.config/resh.toml ]; then - echo "Copying config file ..." - cp -f conf/config.toml ~/.config/resh.toml -# else - # echo "Merging config files ..." - # NOTE: This is where we will merge configs when we make changes to the upstream config - # HINT: check which version are we updating FROM and make changes to config based on that -fi - -echo "Generating completions ..." -bin/resh-control completion bash > ~/.resh/bash_completion.d/_reshctl -bin/resh-control completion zsh > ~/.resh/zsh_completion.d/_reshctl - echo "Copying more files ..." cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid -cp -f bin/* ~/.resh/bin/ -cp -f scripts/resh-evaluate-plot.py ~/.resh/bin/ -cp -fr data/sanitizer ~/.resh/sanitizer_data +cp -f bin/resh-{daemon,cli,control,collect,postcollect,session-init,config} ~/.resh/bin/ -# backward compatibility: We have a new location for resh history file -[ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json +echo "Creating/updating config file ..." +./bin/resh-install-utils migrate-config echo "Finishing up ..." # Adding resh shellrc to .bashrc ... @@ -158,20 +142,21 @@ if [ ! -f ~/.bashrc ]; then touch ~/.bashrc fi grep -q '[[ -f ~/.resh/shellrc ]] && source ~/.resh/shellrc' ~/.bashrc ||\ - echo -e '\n[[ -f ~/.resh/shellrc ]] && source ~/.resh/shellrc # this line was added by RESH (Rich Enchanced Shell History)' >> ~/.bashrc + echo -e '\n[[ -f ~/.resh/shellrc ]] && source ~/.resh/shellrc # this line was added by RESH (Rich Enhanced Shell History)' >> ~/.bashrc # Adding bash-preexec to .bashrc ... grep -q '[[ -f ~/.bash-preexec.sh ]] && source ~/.bash-preexec.sh' ~/.bashrc ||\ - echo -e '\n[[ -f ~/.bash-preexec.sh ]] && source ~/.bash-preexec.sh # this line was added by RESH (Rich Enchanced Shell History)' >> ~/.bashrc + echo -e '\n[[ -f ~/.bash-preexec.sh ]] && source ~/.bash-preexec.sh # this line was added by RESH (Rich Enhanced Shell History)' >> ~/.bashrc # Adding resh shellrc to .zshrc ... if [ -f ~/.zshrc ]; then grep -q '[ -f ~/.resh/shellrc ] && source ~/.resh/shellrc' ~/.zshrc ||\ - echo -e '\n[ -f ~/.resh/shellrc ] && source ~/.resh/shellrc # this line was added by RESH (Rich Enchanced Shell History)' >> ~/.zshrc + echo -e '\n[ -f ~/.resh/shellrc ] && source ~/.resh/shellrc # this line was added by RESH (Rich Enhanced Shell History)' >> ~/.zshrc fi # Deleting zsh completion cache - for future use # [ ! -e ~/.zcompdump ] || rm ~/.zcompdump # Final touch +# TODO: change touch ~/.resh_history.json # Generating resh-uuid ... @@ -190,8 +175,6 @@ if [ -f ~/.resh/resh.pid ]; then else pkill -SIGTERM "resh-daemon" || true fi -# daemon uses xdg path variables -__resh_set_xdg_home_paths __resh_run_daemon @@ -206,6 +189,7 @@ info="---- Scroll down using arrow keys ---- ##################################### " +# FIMXE: update info - resh history path info="$info RESH SEARCH APPLICATION = Redesigned reverse search that actually works @@ -216,20 +200,16 @@ RESH SEARCH APPLICATION = Redesigned reverse search that actually works Host, directories, git remote, and exit status is used to display relevant results first. At first, the search application will use the standard shell history without context. - All history recorded from now on will have context which will by the RESH SEARCH app. - - Enable/disable Ctrl+R binding using reshctl command: - $ reshctl enable ctrl_r_binding - $ reshctl disable ctrl_r_binding + All history recorded from now on will have context which will be used by the RESH SEARCH app. CHECK FOR UPDATES To check for (and install) updates use reshctl command: $ reshctl update HISTORY - Your resh history will be recorded to '~/.resh_history.json' + Your resh history will be recorded to '${XDG_DATA_HOME-~/.local/share}/resh/history/.reshjson' Look at it using e.g. following command (you might need to install jq) - $ tail -f ~/.resh_history.json | jq + $ cat ${XDG_DATA_HOME-~/.local/share}/resh/history/.reshjson | sed 's/^v[^{]*{/{/' | jq . ISSUES & FEEDBACK Please report issues to: https://github.com/curusarn/resh/issues @@ -254,7 +234,7 @@ echo "All done!" echo "Thank you for using RESH" echo "Issues go here: https://github.com/curusarn/resh/issues" echo "Ctrl+R launches the RESH SEARCH app" -# echo "Do not forget to restart your terminal" + if [ -z "${__RESH_VERSION:-}" ]; then echo " ############################################################## # # diff --git a/scripts/resh-evaluate-plot.py b/scripts/resh-evaluate-plot.py deleted file mode 100755 index 89792cb..0000000 --- a/scripts/resh-evaluate-plot.py +++ /dev/null @@ -1,1218 +0,0 @@ -#!/usr/bin/env python3 - - -import traceback -import sys -import json -from collections import defaultdict -import numpy as np -from graphviz import Digraph -from datetime import datetime - -from matplotlib import rcParams -rcParams['font.family'] = 'serif' -# rcParams['font.serif'] = [''] - -import matplotlib.pyplot as plt -import matplotlib.path as mpath -import matplotlib.patches as mpatches - -PLOT_WIDTH = 10 # inches -PLOT_HEIGHT = 7 # inches - -PLOT_SIZE_zipf = 20 - -data = json.load(sys.stdin) - -DATA_records = [] -DATA_records_by_session = defaultdict(list) -DATA_records_by_user = defaultdict(list) -for user in data["UsersRecords"]: - if user["Devices"] is None: - continue - for device in user["Devices"]: - if device["Records"] is None: - continue - for record in device["Records"]: - if "invalid" in record and record["invalid"]: - continue - - DATA_records.append(record) - DATA_records_by_session[record["seqSessionId"]].append(record) - DATA_records_by_user[user["Name"] + ":" + device["Name"]].append(record) - -DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"])) - -for pid, session in DATA_records_by_session.items(): - session = list(sorted(session, key=lambda x: x["realtimeAfterLocal"])) - -# TODO: this should be a cmdline option -async_draw = True - -# for strategy in data["Strategies"]: -# print(json.dumps(strategy)) - -def zipf(length): - return list(map(lambda x: 1/2**x, range(0, length))) - - -def trim(text, length, add_elipse=True): - if add_elipse and len(text) > length: - return text[:length-1] + "…" - return text[:length] - - -# Figure 3.1. The normalized command frequency, compared with Zipf. -def plot_cmdLineFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): - cmdLine_count = defaultdict(int) - for record in DATA_records: - cmdLine_count[record["cmdLine"]] += 1 - - tmp = sorted(cmdLine_count.items(), key=lambda x: x[1], reverse=True)[:plotSize] - cmdLineFrq = list(map(lambda x: x[1] / tmp[0][1], tmp)) - labels = list(map(lambda x: trim(x[0], 7), tmp)) - - ranks = range(1, len(cmdLineFrq)+1) - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.plot(ranks, zipf(len(ranks)), '-') - plt.plot(ranks, cmdLineFrq, 'o-') - plt.title("Commandline frequency / rank") - plt.ylabel("Normalized commandline frequency") - plt.xlabel("Commandline rank") - plt.legend(("Zipf", "Commandline"), loc="best") - if show_labels: - plt.xticks(ranks, labels, rotation=-60) - # TODO: make xticks integral - if async_draw: - plt.draw() - else: - plt.show() - - -# similar to ~ Figure 3.1. The normalized command frequency, compared with Zipf. -def plot_cmdFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command frequency / rank") - plt.ylabel("Normalized command frequency") - plt.xlabel("Command rank") - legend = [] - - - cmd_count = defaultdict(int) - len_records = 0 - for record in DATA_records: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - len_records += 1 - - tmp = sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)[:plotSize] - cmdFrq = list(map(lambda x: x[1] / tmp[0][1], tmp)) - labels = list(map(lambda x: trim(x[0], 7), tmp)) - - top100percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(1 * len(cmd_count))])) / len_records - top10percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.1 * len(cmd_count))])) / len_records - top20percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.2 * len(cmd_count))])) / len_records - print("% ALL: Top {} %% of cmds amounts for {} %% of all command lines".format(100, top100percent)) - print("% ALL: Top {} %% of cmds amounts for {} %% of all command lines".format(10, top10percent)) - print("% ALL: Top {} %% of cmds amounts for {} %% of all command lines".format(20, top20percent)) - ranks = range(1, len(cmdFrq)+1) - plt.plot(ranks, zipf(len(ranks)), '-') - legend.append("Zipf distribution") - plt.plot(ranks, cmdFrq, 'o-') - legend.append("All subjects") - - - for user in DATA_records_by_user.items(): - cmd_count = defaultdict(int) - len_records = 0 - name, records = user - for record in records: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - len_records += 1 - - tmp = sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)[:plotSize] - cmdFrq = list(map(lambda x: x[1] / tmp[0][1], tmp)) - labels = list(map(lambda x: trim(x[0], 7), tmp)) - - top100percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(1 * len(cmd_count))])) / len_records - top10percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.1 * len(cmd_count))])) / len_records - top20percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.2 * len(cmd_count))])) / len_records - print("% {}: Top {} %% of cmds amounts for {} %% of all command lines".format(name, 100, top100percent)) - print("% {}: Top {} %% of cmds amounts for {} %% of all command lines".format(name, 10, top10percent)) - print("% {}: Top {} %% of cmds amounts for {} %% of all command lines".format(name, 20, top20percent)) - ranks = range(1, len(cmdFrq)+1) - plt.plot(ranks, cmdFrq, 'o-') - legend.append("{} (sanitize!)".format(name)) - - plt.legend(legend, loc="best") - - if show_labels: - plt.xticks(ranks, labels, rotation=-60) - # TODO: make xticks integral - if async_draw: - plt.draw() - else: - plt.show() - -# Figure 3.2. Command vocabulary size vs. the number of command lines entered for four individuals. -def plot_cmdVocabularySize_cmdLinesEntered(): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command vocabulary size vs. the number of command lines entered") - plt.ylabel("Command vocabulary size") - plt.xlabel("# of command lines entered") - legend = [] - - # x_count = max(map(lambda x: len(x[1]), DATA_records_by_user.items())) - # x_values = range(0, x_count) - for user in DATA_records_by_user.items(): - new_cmds_after_1k = 0 - new_cmds_after_2k = 0 - new_cmds_after_3k = 0 - cmd_vocabulary = set() - y_cmd_count = [0] - name, records = user - for record in records: - cmd = record["command"] - if cmd == "": - continue - if cmd in cmd_vocabulary: - # repeat last value - y_cmd_count.append(y_cmd_count[-1]) - else: - cmd_vocabulary.add(cmd) - # append last value +1 - y_cmd_count.append(y_cmd_count[-1] + 1) - if len(y_cmd_count) > 1000: - new_cmds_after_1k+=1 - if len(y_cmd_count) > 2000: - new_cmds_after_2k+=1 - if len(y_cmd_count) > 3000: - new_cmds_after_3k+=1 - - if len(y_cmd_count) == 1000: - print("% {}: Cmd adoption rate at 1k (between 0 and 1k) cmdlines = {}".format(name ,len(cmd_vocabulary) / (len(y_cmd_count)))) - if len(y_cmd_count) == 2000: - print("% {}: Cmd adoption rate at 2k cmdlines = {}".format(name ,len(cmd_vocabulary) / (len(y_cmd_count)))) - print("% {}: Cmd adoption rate between 1k and 2k cmdlines = {}".format(name ,new_cmds_after_1k / (len(y_cmd_count) - 1000))) - if len(y_cmd_count) == 3000: - print("% {}: Cmd adoption rate between 2k and 3k cmdlines = {}".format(name ,new_cmds_after_2k / (len(y_cmd_count) - 2000))) - - print("% {}: New cmd adoption rate after 1k cmdlines = {}".format(name ,new_cmds_after_1k / (len(y_cmd_count) - 1000))) - print("% {}: New cmd adoption rate after 2k cmdlines = {}".format(name ,new_cmds_after_2k / (len(y_cmd_count) - 2000))) - print("% {}: New cmd adoption rate after 3k cmdlines = {}".format(name ,new_cmds_after_3k / (len(y_cmd_count) - 3000))) - x_cmds_entered = range(0, len(y_cmd_count)) - plt.plot(x_cmds_entered, y_cmd_count, '-') - legend.append(name + " (TODO: sanitize!)") - - # print(cmd_vocabulary) - - plt.legend(legend, loc="best") - - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_cmdVocabularySize_daily(): - SECONDS_IN_A_DAY = 86400 - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command vocabulary size in days") - plt.ylabel("Command vocabulary size") - plt.xlabel("Days") - legend = [] - - # x_count = max(map(lambda x: len(x[1]), DATA_records_by_user.items())) - # x_values = range(0, x_count) - for user in DATA_records_by_user.items(): - new_cmds_after_100 = 0 - new_cmds_after_200 = 0 - new_cmds_after_300 = 0 - cmd_vocabulary = set() - y_cmd_count = [0] - name, records = user - - cmd_fail_count = 0 - - if not len(records): - print("ERROR: no records for user {}".format(name)) - continue - - first_day = records[0]["realtimeAfter"] - this_day = first_day - - for record in records: - cmd = record["command"] - timestamp = record["realtimeAfter"] - - if cmd == "": - cmd_fail_count += 1 - continue - - if timestamp >= this_day + SECONDS_IN_A_DAY: - this_day += SECONDS_IN_A_DAY - while timestamp >= this_day + SECONDS_IN_A_DAY: - y_cmd_count.append(-10) - this_day += SECONDS_IN_A_DAY - - y_cmd_count.append(len(cmd_vocabulary)) - cmd_vocabulary = set() # wipes the vocabulary each day - - if len(y_cmd_count) > 100: - new_cmds_after_100+=1 - if len(y_cmd_count) > 200: - new_cmds_after_200+=1 - if len(y_cmd_count) > 300: - new_cmds_after_300+=1 - - if len(y_cmd_count) == 100: - print("% {}: Cmd adoption rate at 100 days (between 0 and 100 days) = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - if len(y_cmd_count) == 200: - print("% {}: Cmd adoption rate at 200 days days = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - print("% {}: Cmd adoption rate between 100 and 200 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - if len(y_cmd_count) == 300: - print("% {}: Cmd adoption rate between 200 and 300 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - - if cmd not in cmd_vocabulary: - cmd_vocabulary.add(cmd) - - - print("% {}: New cmd adoption rate after 100 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - print("% {}: New cmd adoption rate after 200 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - print("% {}: New cmd adoption rate after 300 days = {}".format(name, new_cmds_after_300 / (len(y_cmd_count) - 300))) - print("% {}: cmd_fail_count = {}".format(name, cmd_fail_count)) - x_cmds_entered = range(0, len(y_cmd_count)) - plt.plot(x_cmds_entered, y_cmd_count, 'o', markersize=2) - legend.append(name + " (TODO: sanitize!)") - - # print(cmd_vocabulary) - - plt.legend(legend, loc="best") - plt.ylim(bottom=-5) - - if async_draw: - plt.draw() - else: - plt.show() - - -def matplotlib_escape(ss): - ss = ss.replace('$', '\\$') - return ss - - -def plot_cmdUsage_in_time(sort_cmds=False, num_cmds=None): - SECONDS_IN_A_DAY = 86400 - tab_colors = ("tab:blue", "tab:orange", "tab:green", "tab:red", "tab:purple", "tab:brown", "tab:pink", "tab:gray") - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command use in time") - plt.ylabel("Commands") - plt.xlabel("Days") - legend_patches = [] - - cmd_ids = {} - y_labels = [] - - all_x_values = [] - all_y_values = [] - all_s_values = [] # size - all_c_values = [] # color - - x_values = [] - y_values = [] - s_values = [] # size - c_values = [] # color - - if sort_cmds: - cmd_count = defaultdict(int) - for user in DATA_records_by_user.items(): - name, records = user - for record in records: - cmd = record["command"] - cmd_count[cmd] += 1 - - sorted_cmds = map(lambda x: x[0], sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)) - - for cmd in sorted_cmds: - cmd_ids[cmd] = len(cmd_ids) - y_labels.append(matplotlib_escape(cmd)) - - - for user_idx, user in enumerate(DATA_records_by_user.items()): - name, records = user - - if not len(records): - print("ERROR: no records for user {}".format(name)) - continue - - - first_day = records[0]["realtimeAfter"] - this_day = first_day - day_no = 0 - today_cmds = defaultdict(int) - - for record in records: - cmd = record["command"] - timestamp = record["realtimeAfter"] - - if cmd == "": - print("NOTICE: Empty cmd for {}".format(record["cmdLine"])) - continue - - if timestamp >= this_day + SECONDS_IN_A_DAY: - for item in today_cmds.items(): - cmd, count = item - cmd_id = cmd_ids[cmd] - # skip commands with high ids - if num_cmds is not None and cmd_id >= num_cmds: - continue - - x_values.append(day_no) - y_values.append(cmd_id) - s_values.append(count) - c_values.append(tab_colors[user_idx]) - - today_cmds = defaultdict(int) - - this_day += SECONDS_IN_A_DAY - day_no += 1 - while timestamp >= this_day + SECONDS_IN_A_DAY: - this_day += SECONDS_IN_A_DAY - day_no += 1 - - if cmd not in cmd_ids: - cmd_ids[cmd] = len(cmd_ids) - y_labels.append(matplotlib_escape(cmd)) - - today_cmds[cmd] += 1 - - all_x_values.extend(x_values) - all_y_values.extend(y_values) - all_s_values.extend(s_values) - all_c_values.extend(c_values) - x_values = [] - y_values = [] - s_values = [] - c_values = [] - legend_patches.append(mpatches.Patch(color=tab_colors[user_idx], label="{} ({}) (TODO: sanitize!)".format(name, user_idx))) - - if num_cmds is not None and len(y_labels) > num_cmds: - y_labels = y_labels[:num_cmds] - plt.yticks(ticks=range(0, len(y_labels)), labels=y_labels, fontsize=6) - plt.scatter(all_x_values, all_y_values, s=all_s_values, c=all_c_values, marker='o') - plt.legend(handles=legend_patches, loc="best") - - if async_draw: - plt.draw() - else: - plt.show() - - -# Figure 5.6. Command line vocabulary size vs. the number of commands entered for four typical individuals. -def plot_cmdVocabularySize_time(): - SECONDS_IN_A_DAY = 86400 - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command vocabulary size growth in time") - plt.ylabel("Command vocabulary size") - plt.xlabel("Days") - legend = [] - - # x_count = max(map(lambda x: len(x[1]), DATA_records_by_user.items())) - # x_values = range(0, x_count) - for user in DATA_records_by_user.items(): - new_cmds_after_100 = 0 - new_cmds_after_200 = 0 - new_cmds_after_300 = 0 - cmd_vocabulary = set() - y_cmd_count = [0] - name, records = user - - cmd_fail_count = 0 - - if not len(records): - print("ERROR: no records for user {}".format(name)) - continue - - first_day = records[0]["realtimeAfter"] - this_day = first_day - - for record in records: - cmd = record["command"] - timestamp = record["realtimeAfter"] - - if cmd == "": - cmd_fail_count += 1 - continue - - if timestamp >= this_day + SECONDS_IN_A_DAY: - this_day += SECONDS_IN_A_DAY - while timestamp >= this_day + SECONDS_IN_A_DAY: - y_cmd_count.append(-10) - this_day += SECONDS_IN_A_DAY - - y_cmd_count.append(len(cmd_vocabulary)) - - if len(y_cmd_count) > 100: - new_cmds_after_100+=1 - if len(y_cmd_count) > 200: - new_cmds_after_200+=1 - if len(y_cmd_count) > 300: - new_cmds_after_300+=1 - - if len(y_cmd_count) == 100: - print("% {}: Cmd adoption rate at 100 days (between 0 and 100 days) = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - if len(y_cmd_count) == 200: - print("% {}: Cmd adoption rate at 200 days days = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - print("% {}: Cmd adoption rate between 100 and 200 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - if len(y_cmd_count) == 300: - print("% {}: Cmd adoption rate between 200 and 300 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - - if cmd not in cmd_vocabulary: - cmd_vocabulary.add(cmd) - - - print("% {}: New cmd adoption rate after 100 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - print("% {}: New cmd adoption rate after 200 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - print("% {}: New cmd adoption rate after 300 days = {}".format(name, new_cmds_after_300 / (len(y_cmd_count) - 300))) - print("% {}: cmd_fail_count = {}".format(name, cmd_fail_count)) - x_cmds_entered = range(0, len(y_cmd_count)) - plt.plot(x_cmds_entered, y_cmd_count, 'o', markersize=2) - legend.append(name + " (TODO: sanitize!)") - - # print(cmd_vocabulary) - - plt.legend(legend, loc="best") - plt.ylim(bottom=0) - - if async_draw: - plt.draw() - else: - plt.show() - - -# Figure 5.6. Command line vocabulary size vs. the number of commands entered for four typical individuals. -def plot_cmdLineVocabularySize_cmdLinesEntered(): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command line vocabulary size vs. the number of command lines entered") - plt.ylabel("Command line vocabulary size") - plt.xlabel("# of command lines entered") - legend = [] - - for user in DATA_records_by_user.items(): - cmdLine_vocabulary = set() - y_cmdLine_count = [0] - name, records = user - for record in records: - cmdLine = record["cmdLine"] - if cmdLine in cmdLine_vocabulary: - # repeat last value - y_cmdLine_count.append(y_cmdLine_count[-1]) - else: - cmdLine_vocabulary.add(cmdLine) - # append last value +1 - y_cmdLine_count.append(y_cmdLine_count[-1] + 1) - - # print(cmdLine_vocabulary) - x_cmdLines_entered = range(0, len(y_cmdLine_count)) - plt.plot(x_cmdLines_entered, y_cmdLine_count, '-') - legend.append(name + " (TODO: sanitize!)") - - plt.legend(legend, loc="best") - - if async_draw: - plt.draw() - else: - plt.show() - -# Figure 3.3. Sequential structure of UNIX command usage, from Figure 4 in Hanson et al. (1984). -# Ball diameters are proportional to stationary probability. Lines indicate significant dependencies, -# solid ones being more probable (p < .0001) and dashed ones less probable (.005 < p < .0001). -def graph_cmdSequences(node_count=33, edge_minValue=0.05, view_graph=True): - START_CMD = "_start_" - END_CMD = "_end_" - cmd_count = defaultdict(int) - cmdSeq_count = defaultdict(lambda: defaultdict(int)) - cmd_id = dict() - x = 0 - cmd_id[START_CMD] = str(x) - x += 1 - cmd_id[END_CMD] = str(x) - for pid, session in DATA_records_by_session.items(): - cmd_count[START_CMD] += 1 - prev_cmd = START_CMD - for record in session: - cmd = record["command"] - if cmd == "": - continue - cmdSeq_count[prev_cmd][cmd] += 1 - cmd_count[cmd] += 1 - if cmd not in cmd_id: - x += 1 - cmd_id[cmd] = str(x) - prev_cmd = cmd - # end the session - cmdSeq_count[prev_cmd][END_CMD] += 1 - cmd_count[END_CMD] += 1 - - - # get `node_count` of largest nodes - sorted_cmd_count = sorted(cmd_count.items(), key=lambda x: x[1], reverse=True) - print(sorted_cmd_count) - cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:node_count] - - # use 3 biggest nodes as a reference point for scaling - biggest_node = cmd_count[cmds_to_graph[0]] - nd_biggest_node = cmd_count[cmds_to_graph[1]] - rd_biggest_node = cmd_count[cmds_to_graph[1]] - count2scale_coef = 3 / (biggest_node + nd_biggest_node + rd_biggest_node) - - # scaling constant - # affects node size and node label - base_scaling_factor = 21 - # extra scaling for experiments - not really useful imho - # affects everything nodes, edges, node labels, treshold for turning label into xlabel, xlabel size, ... - extra_scaling_factor = 1.0 - for x in range(0, 10): - # graphviz is not the most reliable piece of software - # -> retry on fail but scale nodes down by 1% - scaling_factor = base_scaling_factor * (1 - x * 0.01) - - # overlap: scale -> solve overlap by scaling the graph - # overlap_shrink -> try to shrink the graph a bit after you are done - # splines -> don't draw edges over nodes - # sep: 2.5 -> assume that nodes are 2.5 inches larger - graph_attr={'overlap':'scale', 'overlap_shrink':'true', - 'splines':'true', 'sep':'0.25'} - graph = Digraph(name='command_sequentiality', engine='neato', graph_attr=graph_attr) - - # iterate over all nodes - for cmd in cmds_to_graph: - seq = cmdSeq_count[cmd] - count = cmd_count[cmd] - - # iterate over all "following" commands (for each node) - for seq_entry in seq.items(): - cmd2, seq_count = seq_entry - relative_seq_count = seq_count / count - - # check if "follow" command is supposed to be in the graph - if cmd2 not in cmds_to_graph: - continue - # check if the edge value is high enough - if relative_seq_count < edge_minValue: - continue - - # create starting node and end node for the edge - # duplicates don't matter - for id_, cmd_ in ((cmd_id[cmd], cmd), (cmd_id[cmd2], cmd2)): - count_ = cmd_count[cmd_] - scale_ = count_ * count2scale_coef * scaling_factor * extra_scaling_factor - width_ = 0.08 * scale_ - fontsize_ = 8.5 * scale_ / (len(cmd_) + 3) - - width_ = str(width_) - if fontsize_ < 12 * extra_scaling_factor: - graph.node(id_, ' ', shape='circle', fixedsize='true', fontname='monospace bold', - width=width_, fontsize=str(12 * extra_scaling_factor), forcelabels='true', xlabel=cmd_) - else: - fontsize_ = str(fontsize_) - graph.node(id_, cmd_, shape='circle', fixedsize='true', fontname='monospace bold', - width=width_, fontsize=fontsize_, forcelabels='true', labelloc='c') - - # value of the edge (percentage) 1.0 is max - scale_ = seq_count / cmd_count[cmd] - penwidth_ = str((0.5 + 4.5 * scale_) * extra_scaling_factor) - #penwidth_bold_ = str(8 * scale_) - # if scale_ > 0.5: - # graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='true', splines='curved', - # penwidth=penwidth_, style='bold', arrowhead='diamond') - # elif scale_ > 0.2: - if scale_ > 0.3: - scale_ = str(int(scale_ * 100)/100) - graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='true', splines='curved', - penwidth=penwidth_, forcelables='true', label=scale_) - elif scale_ > 0.2: - graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='true', splines='curved', - penwidth=penwidth_, style='dashed') - # elif scale_ > 0.1: - else: - graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='false', splines='curved', - penwidth=penwidth_, style='dotted', arrowhead='empty') - - # graphviz sometimes fails - see above - try: - # graph.view() - graph.render('/tmp/resh-graph-command_sequence-nodeCount_{}-edgeMinVal_{}.gv'.format(node_count, edge_minValue), view=view_graph) - break - except Exception as e: - trace = traceback.format_exc() - print("GRAPHVIZ EXCEPTION: <{}>\nGRAPHVIZ TRACE: <{}>".format(str(e), trace)) - - -def plot_strategies_matches(plot_size=50, selected_strategies=[], show_strat_title=True, force_strat_title=None): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Matches at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel('%' + " of matches") - plt.xlabel("Distance") - legend = [] - x_values = range(1, plot_size+1) - saved_matches_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_matches_total = matches_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - acc = 0 - matches_cumulative = [] - for x in matches: - acc += x - matches_cumulative.append(acc) - # matches_cumulative.append(matches_total) - matches_percent = list(map(lambda x: 100 * x / dataPoint_count, matches_cumulative)) - - plt.plot(x_values, matches_percent, 'o-') - if force_strat_title is not None: - legend.append(force_strat_title) - else: - legend.append(strategy_title) - - - assert(saved_matches_total is not None) - assert(saved_dataPoint_count is not None) - max_values = [100 * saved_matches_total / saved_dataPoint_count] * len(x_values) - print("% >>> Avg recurrence rate = {}".format(max_values[0])) - plt.plot(x_values, max_values, 'r-') - legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - acc = 0 - charsRecalled_cumulative = [] - for x in charsRecalled: - acc += x - charsRecalled_cumulative.append(acc) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - print("% >>> Max avg recalled characters = {}".format(max_values[0])) - plt.plot(x_values, max_values, 'r-') - legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - plt.ylim(bottom=0) - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance (including prefix matches) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled (including prefix matches)") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for multiMatch in strategy["PrefixMatches"]: - dataPoint_count += 1 - - if not multiMatch["Match"]: - continue - matches_total += 1 - - last_charsRecalled = 0 - for match in multiMatch["Entries"]: - - chars = match["CharsRecalled"] - charsIncrease = chars - last_charsRecalled - assert(charsIncrease > 0) - charsRecalled_total += charsIncrease - - dist = match["Distance"] - if dist > plot_size: - continue - - charsRecalled[dist-1] += charsIncrease - last_charsRecalled = chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - acc = 0 - charsRecalled_cumulative = [] - for x in charsRecalled: - acc += x - charsRecalled_cumulative.append(acc) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - print("% >>> Max avg recalled characters (including prefix matches) = {}".format(max_values[0])) - plt.plot(x_values, max_values, 'r-') - legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - plt.ylim(bottom=0) - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_matches_noncummulative(plot_size=50, selected_strategies=["recent (bash-like)"], show_strat_title=False, force_strat_title=None): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Matches at distance (noncumulative) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel('%' + " of matches") - plt.xlabel("Distance") - legend = [] - x_values = range(1, plot_size+1) - saved_matches_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_matches_total = matches_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - # acc = 0 - # matches_cumulative = [] - # for x in matches: - # acc += x - # matches_cumulative.append(acc) - # # matches_cumulative.append(matches_total) - matches_percent = list(map(lambda x: 100 * x / dataPoint_count, matches)) - - plt.plot(x_values, matches_percent, 'o-') - if force_strat_title is not None: - legend.append(force_strat_title) - else: - legend.append(strategy_title) - - assert(saved_matches_total is not None) - assert(saved_dataPoint_count is not None) - # max_values = [100 * saved_matches_total / saved_dataPoint_count] * len(x_values) - # print("% >>> Avg recurrence rate = {}".format(max_values[0])) - # plt.plot(x_values, max_values, 'r-') - # legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - # plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled_noncummulative(plot_size=50, selected_strategies=["recent (bash-like)"], show_strat_title=False): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance (noncumulative) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - # acc = 0 - # charsRecalled_cumulative = [] - # for x in charsRecalled: - # acc += x - # charsRecalled_cumulative.append(acc) - # charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - # max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - # print("% >>> Max avg recalled characters = {}".format(max_values[0])) - # plt.plot(x_values, max_values, 'r-') - # legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - # plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled_prefix_noncummulative(plot_size=50, selected_strategies=["recent (bash-like)"], show_strat_title=False): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance (including prefix matches) (noncummulative) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled (including prefix matches)") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for multiMatch in strategy["PrefixMatches"]: - dataPoint_count += 1 - - if not multiMatch["Match"]: - continue - matches_total += 1 - - last_charsRecalled = 0 - for match in multiMatch["Entries"]: - - chars = match["CharsRecalled"] - charsIncrease = chars - last_charsRecalled - assert(charsIncrease > 0) - charsRecalled_total += charsIncrease - - dist = match["Distance"] - if dist > plot_size: - continue - - charsRecalled[dist-1] += charsIncrease - last_charsRecalled = chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - # acc = 0 - # charsRecalled_cumulative = [] - # for x in charsRecalled: - # acc += x - # charsRecalled_cumulative.append(acc) - # charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - # max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - # print("% >>> Max avg recalled characters (including prefix matches) = {}".format(max_values[0])) - # plt.plot(x_values, max_values, 'r-') - # legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - # plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def print_top_cmds(num_cmds=20): - cmd_count = defaultdict(int) - cmd_total = 0 - for pid, session in DATA_records_by_session.items(): - for record in session: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - cmd_total += 1 - - # get `node_count` of largest nodes - sorted_cmd_count = list(sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)) - print("\n\n% All subjects: Top commands") - for cmd, count in sorted_cmd_count[:num_cmds]: - print("{} {}".format(cmd, count)) - # print(sorted_cmd_count) - # cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:cmd_count] - - -def print_top_cmds_by_user(num_cmds=20): - for user in DATA_records_by_user.items(): - name, records = user - cmd_count = defaultdict(int) - cmd_total = 0 - for record in records: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - cmd_total += 1 - - # get `node_count` of largest nodes - sorted_cmd_count = list(sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)) - print("\n\n% {}: Top commands".format(name)) - for cmd, count in sorted_cmd_count[:num_cmds]: - print("{} {}".format(cmd, count)) - # print(sorted_cmd_count) - # cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:cmd_count] - - -def print_avg_cmdline_length(): - cmd_len_total = 0 - cmd_total = 0 - for pid, session in DATA_records_by_session.items(): - for record in session: - cmd = record["cmdLine"] - if cmd == "": - continue - cmd_len_total += len(cmd) - cmd_total += 1 - - print("% ALL avg cmdline = {}".format(cmd_len_total / cmd_total)) - # print(sorted_cmd_count) - # cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:cmd_count] - - -# plot_cmdLineFrq_rank() -# plot_cmdFrq_rank() -print_top_cmds(30) -print_top_cmds_by_user(30) -# print_avg_cmdline_length() -# -# plot_cmdLineVocabularySize_cmdLinesEntered() -plot_cmdVocabularySize_cmdLinesEntered() -plot_cmdVocabularySize_time() -# plot_cmdVocabularySize_daily() -plot_cmdUsage_in_time(num_cmds=100) -plot_cmdUsage_in_time(sort_cmds=True, num_cmds=100) -# -recent_strats=("recent", "recent (bash-like)") -recurrence_strat=("recent (bash-like)",) -# plot_strategies_matches(20) -# plot_strategies_charsRecalled(20) -# plot_strategies_charsRecalled_prefix(20) -# plot_strategies_charsRecalled_noncummulative(20, selected_strategies=recent_strats) -# plot_strategies_matches_noncummulative(20) -# plot_strategies_charsRecalled_noncummulative(20) -# plot_strategies_charsRecalled_prefix_noncummulative(20) -# plot_strategies_matches(20, selected_strategies=recurrence_strat, show_strat_title=True, force_strat_title="recurrence rate") -# plot_strategies_matches_noncummulative(20, selected_strategies=recurrence_strat, show_strat_title=True, force_strat_title="recurrence rate") - -# graph_cmdSequences(node_count=33, edge_minValue=0.048) - -# graph_cmdSequences(node_count=28, edge_minValue=0.06) - -# new improved -# for n in range(40, 43): -# for e in range(94, 106, 2): -# e *= 0.001 -# graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False) - -#for n in range(29, 35): -# for e in range(44, 56, 2): -# e *= 0.001 -# graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False) - -# be careful and check if labels fit the display - -if async_draw: - plt.show() diff --git a/scripts/reshctl.sh b/scripts/reshctl.sh index db02147..69ab09d 100644 --- a/scripts/reshctl.sh +++ b/scripts/reshctl.sh @@ -1,3 +1,4 @@ +#!/hint/sh # shellcheck source=../submodules/bash-zsh-compat-widgets/bindfunc.sh . ~/.resh/bindfunc.sh @@ -73,83 +74,6 @@ resh() { elif [ $status_code = 130 ]; then true else - local fpath_last_run="$__RESH_XDG_CACHE_HOME/cli_last_run_out.txt" - echo "$buffer" >| "$fpath_last_run" - echo "resh-cli failed - check '$fpath_last_run' and '~/.resh/cli.log'" + printf "%s" "$buffer" >&2 fi -} - -reshctl() { - # export current shell because resh-control needs to know - export __RESH_ctl_shell=$__RESH_SHELL - # run resh-control aka the real reshctl - resh-control "$@" - - # modify current shell session based on exit status - local _status=$? - # echo $_status - # unexport current shell - unset __RESH_ctl_shell - case "$_status" in - 0|1) - # success | fail - return "$_status" - ;; - # enable - # 30) - # # enable all - # __resh_bind_all - # return 0 - # ;; - 32) - # enable control R - __resh_bind_control_R - return 0 - ;; - # disable - # 40) - # # disable all - # __resh_unbind_all - # return 0 - # ;; - 42) - # disable control R - __resh_unbind_control_R - return 0 - ;; - 50) - # reload rc files - . ~/.resh/shellrc - return 0 - ;; - 51) - # inspect session history - # reshctl debug inspect N - resh-inspect --sessionID "$__RESH_SESSION_ID" --count "${3-10}" - return 0 - ;; - 52) - # show status - echo - echo 'Control R binding ...' - if [ "$(resh-config --key BindControlR)" = true ]; then - echo ' * future sessions: ENABLED' - else - echo ' * future sessions: DISABLED' - fi - if [ "${__RESH_control_R_bind_enabled-0}" != 0 ]; then - echo ' * this session: ENABLED' - else - echo ' * this session: DISABLED' - fi - return 0 - ;; - *) - echo "reshctl() FATAL ERROR: unknown status ($_status)" >&2 - echo "Possibly caused by version mismatch between installed resh and resh in this session." >&2 - echo "Please REPORT this issue here: https://github.com/curusarn/resh/issues" >&2 - echo "Please RESTART your terminal window." >&2 - return "$_status" - ;; - esac -} +} \ No newline at end of file diff --git a/scripts/shellrc.sh b/scripts/shellrc.sh index c21f335..cf56daf 100644 --- a/scripts/shellrc.sh +++ b/scripts/shellrc.sh @@ -1,3 +1,4 @@ +#!/hint/sh PATH=$PATH:~/.resh/bin # if [ -n "$ZSH_VERSION" ]; then @@ -11,81 +12,45 @@ PATH=$PATH:~/.resh/bin # shellcheck source=reshctl.sh . ~/.resh/reshctl.sh -__RESH_MACOS=0 -__RESH_LINUX=0 -__RESH_UNAME=$(uname) - -if [ "$__RESH_UNAME" = "Darwin" ]; then - __RESH_MACOS=1 -elif [ "$__RESH_UNAME" = "Linux" ]; then - __RESH_LINUX=1 -else - echo "resh PANIC unrecognized OS" -fi - if [ -n "${ZSH_VERSION-}" ]; then # shellcheck disable=SC1009 __RESH_SHELL="zsh" __RESH_HOST="$HOST" - __RESH_HOSTTYPE="$CPUTYPE" - __resh_zsh_completion_init elif [ -n "${BASH_VERSION-}" ]; then __RESH_SHELL="bash" __RESH_HOST="$HOSTNAME" - __RESH_HOSTTYPE="$HOSTTYPE" - __resh_bash_completion_init else - echo "resh PANIC unrecognized shell" + echo "RESH PANIC: unrecognized shell - please report this to https://github.com/curusarn/resh/issues" fi -# posix +# TODO: read this from resh-specific file +# create that file during install +__RESH_DEVICE="$__RESH_HOST" __RESH_HOME="$HOME" -__RESH_LOGIN="$LOGNAME" -__RESH_SHELL_ENV="$SHELL" -__RESH_TERM="$TERM" - -# non-posix -__RESH_RT_SESSION=$(__resh_get_epochrealtime) -__RESH_OSTYPE="$OSTYPE" -__RESH_MACHTYPE="$MACHTYPE" - -if [ $__RESH_LINUX -eq 1 ]; then - __RESH_OS_RELEASE_ID=$(. /etc/os-release; echo "$ID") - __RESH_OS_RELEASE_VERSION_ID=$(. /etc/os-release; echo "$VERSION_ID") - __RESH_OS_RELEASE_ID_LIKE=$(. /etc/os-release; echo "$ID_LIKE") - __RESH_OS_RELEASE_NAME=$(. /etc/os-release; echo "$NAME") - __RESH_OS_RELEASE_PRETTY_NAME=$(. /etc/os-release; echo "$PRETTY_NAME") - __RESH_RT_SESS_SINCE_BOOT=$(cut -d' ' -f1 /proc/uptime) -elif [ $__RESH_MACOS -eq 1 ]; then - __RESH_OS_RELEASE_ID="macos" - __RESH_OS_RELEASE_VERSION_ID=$(sw_vers -productVersion 2>/dev/null) - __RESH_OS_RELEASE_NAME="macOS" - __RESH_OS_RELEASE_PRETTY_NAME="Mac OS X" - __RESH_RT_SESS_SINCE_BOOT=$(sysctl -n kern.boottime | awk '{print $4}' | sed 's/,//g') -fi # shellcheck disable=2155 export __RESH_VERSION=$(resh-collect -version) # shellcheck disable=2155 export __RESH_REVISION=$(resh-collect -revision) -__resh_set_xdg_home_paths - __resh_run_daemon [ "$(resh-config --key BindControlR)" = true ] && __resh_bind_control_R # block for anything we only want to do once per session # NOTE: nested shells are still the same session +# i.e. $__RESH_SESSION_ID will be set in nested shells if [ -z "${__RESH_SESSION_ID+x}" ]; then export __RESH_SESSION_ID; __RESH_SESSION_ID=$(__resh_get_uuid) export __RESH_SESSION_PID="$$" - # TODO add sesson time + __resh_reset_variables __resh_session_init fi # block for anything we only want to do once per shell +# NOTE: nested shells are new shells +# i.e. $__RESH_INIT_DONE will NOT be set in nested shells if [ -z "${__RESH_INIT_DONE+x}" ]; then preexec_functions+=(__resh_preexec) precmd_functions+=(__resh_precmd) diff --git a/scripts/util.sh b/scripts/util.sh index 4746eb0..37d7b1f 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -1,3 +1,5 @@ +#!/hint/sh + # util.sh - resh utility functions __resh_get_uuid() { cat /proc/sys/kernel/random/uuid 2>/dev/null || resh-uuid @@ -45,80 +47,23 @@ __resh_get_epochrealtime() { fi } +# FIXME: figure out if stdout/stderr should be discarded __resh_run_daemon() { if [ -n "${ZSH_VERSION-}" ]; then setopt LOCAL_OPTIONS NO_NOTIFY NO_MONITOR fi - local fpath_last_run="$__RESH_XDG_CACHE_HOME/daemon_last_run_out.txt" if [ "$(uname)" = Darwin ]; then # hotfix - gnohup resh-daemon >| "$fpath_last_run" 2>&1 & disown + gnohup resh-daemon >/dev/null 2>/dev/null & disown else # TODO: switch to nohup for consistency once you confirm that daemon is # not getting killed anymore on macOS - # nohup resh-daemon >| "$fpath_last_run" 2>&1 & disown - setsid resh-daemon >| "$fpath_last_run" 2>&1 & disown - fi -} - -__resh_bash_completion_init() { - # primitive check to find out if bash_completions are installed - # skip completion init if they are not - _get_comp_words_by_ref >/dev/null 2>/dev/null - [[ $? == 127 ]] && return - local bash_completion_dir=~/.resh/bash_completion.d - if [[ -d $bash_completion_dir && -r $bash_completion_dir && \ - -x $bash_completion_dir ]]; then - for i in $(LC_ALL=C command ls "$bash_completion_dir"); do - i=$bash_completion_dir/$i - # shellcheck disable=SC2154 - # shellcheck source=/dev/null - [[ -f "$i" && -r "$i" ]] && . "$i" - done - fi -} - -__resh_zsh_completion_init() { - # NOTE: this is hacky - each completion needs to be added individually - # TODO: fix later - # fpath=(~/.resh/zsh_completion.d $fpath) - # we should be using fpath but that doesn't work well with oh-my-zsh - # so we are just adding it manually - # shellcheck disable=1090 - if typeset -f compdef >/dev/null 2>&1; then - source ~/.resh/zsh_completion.d/_reshctl && compdef _reshctl reshctl - else - # fallback I guess - fpath=(~/.resh/zsh_completion.d $fpath) - __RESH_zsh_no_compdef=1 + nohup resh-daemon >/dev/null 2>/dev/null & disown + #setsid resh-daemon 2>&1 & disown fi - - # TODO: test and use this - # NOTE: this is not how globbing works - # for f in ~/.resh/zsh_completion.d/_*; do - # source ~/.resh/zsh_completion.d/_$f && compdef _$f $f - # done } __resh_session_init() { - # posix - local __RESH_COLS="$COLUMNS" - local __RESH_LANG="$LANG" - local __RESH_LC_ALL="$LC_ALL" - # other LC ? - local __RESH_LINES="$LINES" - local __RESH_PWD="$PWD" - - # non-posix - local __RESH_SHLVL="$SHLVL" - - # pid - local __RESH_PID; __RESH_PID=$(__resh_get_pid) - - # time - local __RESH_TZ_BEFORE; __RESH_TZ_BEFORE=$(date +%z) - local __RESH_RT_BEFORE; __RESH_RT_BEFORE=$(__resh_get_epochrealtime) - if [ "$__RESH_VERSION" != "$(resh-session-init -version)" ]; then # shellcheck source=shellrc.sh source ~/.resh/shellrc @@ -135,63 +80,9 @@ __resh_session_init() { fi fi if [ "$__RESH_VERSION" = "$(resh-session-init -version)" ] && [ "$__RESH_REVISION" = "$(resh-session-init -revision)" ]; then - local fpath_last_run="$__RESH_XDG_CACHE_HOME/session_init_last_run_out.txt" resh-session-init -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ - -shell "$__RESH_SHELL" \ - -uname "$__RESH_UNAME" \ -sessionId "$__RESH_SESSION_ID" \ - -cols "$__RESH_COLS" \ - -home "$__RESH_HOME" \ - -lang "$__RESH_LANG" \ - -lcAll "$__RESH_LC_ALL" \ - -lines "$__RESH_LINES" \ - -login "$__RESH_LOGIN" \ - -shellEnv "$__RESH_SHELL_ENV" \ - -term "$__RESH_TERM" \ - -pid "$__RESH_PID" \ - -sessionPid "$__RESH_SESSION_PID" \ - -host "$__RESH_HOST" \ - -hosttype "$__RESH_HOSTTYPE" \ - -ostype "$__RESH_OSTYPE" \ - -machtype "$__RESH_MACHTYPE" \ - -shlvl "$__RESH_SHLVL" \ - -realtimeBefore "$__RESH_RT_BEFORE" \ - -realtimeSession "$__RESH_RT_SESSION" \ - -realtimeSessSinceBoot "$__RESH_RT_SESS_SINCE_BOOT" \ - -timezoneBefore "$__RESH_TZ_BEFORE" \ - -osReleaseId "$__RESH_OS_RELEASE_ID" \ - -osReleaseVersionId "$__RESH_OS_RELEASE_VERSION_ID" \ - -osReleaseIdLike "$__RESH_OS_RELEASE_ID_LIKE" \ - -osReleaseName "$__RESH_OS_RELEASE_NAME" \ - -osReleasePrettyName "$__RESH_OS_RELEASE_PRETTY_NAME" \ - >| "$fpath_last_run" 2>&1 || echo "resh-session-init ERROR: $(head -n 1 $fpath_last_run)" - fi -} - -__resh_set_xdg_home_paths() { - if [ -z "${XDG_CONFIG_HOME-}" ]; then - __RESH_XDG_CONFIG_FILE="$HOME/.config" - else - __RESH_XDG_CONFIG_FILE="$XDG_CONFIG_HOME" - fi - mkdir -p "$__RESH_XDG_CONFIG_FILE" >/dev/null 2>/dev/null - __RESH_XDG_CONFIG_FILE="$__RESH_XDG_CONFIG_FILE/resh.toml" - - - if [ -z "${XDG_CACHE_HOME-}" ]; then - __RESH_XDG_CACHE_HOME="$HOME/.cache/resh" - else - __RESH_XDG_CACHE_HOME="$XDG_CACHE_HOME/resh" - fi - mkdir -p "$__RESH_XDG_CACHE_HOME" >/dev/null 2>/dev/null - export __RESH_XDG_CACHE_HOME - - - if [ -z "${XDG_DATA_HOME-}" ]; then - __RESH_XDG_DATA_HOME="$HOME/.local/share/resh" - else - __RESH_XDG_DATA_HOME="$XDG_DATA_HOME/resh" + -sessionPid "$__RESH_SESSION_PID" fi - mkdir -p "$__RESH_XDG_DATA_HOME" >/dev/null 2>/dev/null } diff --git a/scripts/widgets.sh b/scripts/widgets.sh index defc605..e8acd55 100644 --- a/scripts/widgets.sh +++ b/scripts/widgets.sh @@ -1,3 +1,4 @@ +#!/hint/sh # shellcheck source=hooks.sh . ~/.resh/hooks.sh @@ -9,17 +10,12 @@ __resh_widget_control_R() { # shellcheck disable=2034 __bp_preexec_interactive_mode="on" - # local __RESH_PREFIX=${BUFFER:0:CURSOR} - # __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;control_R:$__RESH_PREFIX" local PREVBUFFER=$BUFFER - __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS|||control_R:$BUFFER" local status_code local git_remote; git_remote="$(git remote get-url origin 2>/dev/null)" BUFFER=$(resh-cli --sessionID "$__RESH_SESSION_ID" --host "$__RESH_HOST" --pwd "$PWD" --gitOriginRemote "$git_remote" --query "$BUFFER") status_code=$? - local fpath_last_run="$__RESH_XDG_CACHE_HOME/cli_last_run_out.txt" - touch "$fpath_last_run" if [ $status_code = 111 ]; then # execute if [ -n "${ZSH_VERSION-}" ]; then @@ -37,13 +33,11 @@ __resh_widget_control_R() { bind -x '"\u[32~": __resh_nop' fi else - echo "$BUFFER" >| "$fpath_last_run" - echo "# RESH SEARCH APP failed - sorry for the inconvinience - check '$fpath_last_run' and '~/.resh/cli.log'" + echo "RESH SEARCH APP failed" + printf "%s" "$buffer" >&2 BUFFER="$PREVBUFFER" fi CURSOR=${#BUFFER} - # recorded to history - __RESH_HIST_PREV_LINE=${BUFFER} } __resh_widget_control_R_compat() {