From 2c7947225ee22ea71afbbfe088b1c69d03d6f753 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 16 Apr 2022 02:11:59 +0200 Subject: [PATCH 01/35] remove unnecesary commands --- cmd/daemon/run-server.go | 1 + cmd/evaluate/main.go | 152 --------------------------------------- cmd/event/main.go | 7 -- cmd/inspect/main.go | 87 ---------------------- 4 files changed, 1 insertion(+), 246 deletions(-) delete mode 100644 cmd/evaluate/main.go delete mode 100644 cmd/event/main.go delete mode 100644 cmd/inspect/main.go diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index fc70825..9114b20 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -60,6 +60,7 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa mux.HandleFunc("/status", statusHandler) mux.Handle("/record", &recordHandler{subscribers: recordSubscribers}) mux.Handle("/session_init", &sessionInitHandler{subscribers: sessionInitSubscribers}) + // TOOD: drop recall and inspect mux.Handle("/recall", &recallHandler{sesshistDispatch: sesshistDispatch}) mux.Handle("/inspect", &inspectHandler{sesshistDispatch: sesshistDispatch}) mux.Handle("/dump", &dumpHandler{histfileBox: histfileBox}) diff --git a/cmd/evaluate/main.go b/cmd/evaluate/main.go deleted file mode 100644 index 5b4bda8..0000000 --- a/cmd/evaluate/main.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - "os/user" - "path/filepath" - - "github.com/curusarn/resh/pkg/histanal" - "github.com/curusarn/resh/pkg/strat" -) - -// version from git set during build -var version string - -// commit from git set during build -var commit string - -func main() { - const maxCandidates = 50 - - usr, _ := user.Current() - dir := usr.HomeDir - historyPath := filepath.Join(dir, ".resh_history.json") - historyPathBatchMode := filepath.Join(dir, "resh_history.json") - sanitizedHistoryPath := filepath.Join(dir, "resh_history_sanitized.json") - // tmpPath := "/tmp/resh-evaluate-tmp.json" - - showVersion := flag.Bool("version", false, "Show version and exit") - showRevision := flag.Bool("revision", false, "Show git revision and exit") - input := flag.String("input", "", - "Input file (default: "+historyPath+" OR "+sanitizedHistoryPath+ - " depending on --sanitized-input option)") - // outputDir := flag.String("output", "/tmp/resh-evaluate", "Output directory") - sanitizedInput := flag.Bool("sanitized-input", false, - "Handle input as sanitized (also changes default value for input argument)") - plottingScript := flag.String("plotting-script", "resh-evaluate-plot.py", "Script to use for plotting") - inputDataRoot := flag.String("input-data-root", "", - "Input data root, enables batch mode, looks for files matching --input option") - slow := flag.Bool("slow", false, - "Enables strategies that takes a long time (e.g. markov chain strategies).") - skipFailedCmds := flag.Bool("skip-failed-cmds", false, - "Skips records with non-zero exit status.") - debugRecords := flag.Float64("debug", 0, "Debug records - percentage of records that should be debugged.") - - flag.Parse() - - // handle show{Version,Revision} options - if *showVersion == true { - fmt.Println(version) - os.Exit(0) - } - if *showRevision == true { - fmt.Println(commit) - os.Exit(0) - } - - // handle batch mode - batchMode := false - if *inputDataRoot != "" { - batchMode = true - } - // set default input - if *input == "" { - if *sanitizedInput { - *input = sanitizedHistoryPath - } else if batchMode { - *input = historyPathBatchMode - } else { - *input = historyPath - } - } - - var evaluator histanal.HistEval - if batchMode { - evaluator = histanal.NewHistEvalBatchMode(*input, *inputDataRoot, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) - } else { - evaluator = histanal.NewHistEval(*input, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) - } - - var simpleStrategies []strat.ISimpleStrategy - var strategies []strat.IStrategy - - // dummy := strategyDummy{} - // simpleStrategies = append(simpleStrategies, &dummy) - - simpleStrategies = append(simpleStrategies, &strat.Recent{}) - - // frequent := strategyFrequent{} - // frequent.init() - // simpleStrategies = append(simpleStrategies, &frequent) - - // random := strategyRandom{candidatesSize: maxCandidates} - // random.init() - // simpleStrategies = append(simpleStrategies, &random) - - directory := strat.DirectorySensitive{} - directory.Init() - simpleStrategies = append(simpleStrategies, &directory) - - // dynamicDistG := strat.DynamicRecordDistance{ - // MaxDepth: 3000, - // DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1, Git: 10}, - // Label: "10*pwd,10*realpwd,session,time,10*git", - // } - // dynamicDistG.Init() - // strategies = append(strategies, &dynamicDistG) - - // NOTE: this is the decent one !!! - // distanceStaticBest := strat.RecordDistance{ - // MaxDepth: 3000, - // DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, - // Label: "10*pwd,10*realpwd,session,time", - // } - // strategies = append(strategies, &distanceStaticBest) - - recentBash := strat.RecentBash{} - recentBash.Init() - strategies = append(strategies, &recentBash) - - if *slow { - - markovCmd := strat.MarkovChainCmd{Order: 1} - markovCmd.Init() - - markovCmd2 := strat.MarkovChainCmd{Order: 2} - markovCmd2.Init() - - markov := strat.MarkovChain{Order: 1} - markov.Init() - - markov2 := strat.MarkovChain{Order: 2} - markov2.Init() - - simpleStrategies = append(simpleStrategies, &markovCmd2, &markovCmd, &markov2, &markov) - } - - for _, strategy := range simpleStrategies { - strategies = append(strategies, strat.NewSimpleStrategyWrapper(strategy)) - } - - for _, strat := range strategies { - err := evaluator.Evaluate(strat) - if err != nil { - log.Println("Evaluator evaluate() error:", err) - } - } - - evaluator.CalculateStatsAndPlot(*plottingScript) -} diff --git a/cmd/event/main.go b/cmd/event/main.go deleted file mode 100644 index fe3cb72..0000000 --- a/cmd/event/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "fmt" - -func main() { - fmt.Println("Hell world") -} diff --git a/cmd/inspect/main.go b/cmd/inspect/main.go deleted file mode 100644 index 7b76a40..0000000 --- a/cmd/inspect/main.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "net/http" - "time" - - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/msg" - - "os/user" - "path/filepath" - "strconv" -) - -// version from git set during build -var version string - -// commit from git set during build -var commit string - -func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) - } - - sessionID := flag.String("sessionID", "", "resh generated session id") - count := flag.Uint("count", 10, "Number of cmdLines to return") - flag.Parse() - - if *sessionID == "" { - fmt.Println("Error: you need to specify sessionId") - } - - m := msg.InspectMsg{SessionID: *sessionID, Count: *count} - resp := SendInspectMsg(m, strconv.Itoa(config.Port)) - for _, cmdLine := range resp.CmdLines { - fmt.Println("`" + cmdLine + "'") - } -} - -// SendInspectMsg to daemon -func SendInspectMsg(m msg.InspectMsg, port string) msg.MultiResponse { - recJSON, err := json.Marshal(m) - if err != nil { - log.Fatal("send err 1", err) - } - - req, err := http.NewRequest("POST", "http://localhost:"+port+"/inspect", - bytes.NewBuffer(recJSON)) - if err != nil { - log.Fatal("send err 2", err) - } - req.Header.Set("Content-Type", "application/json") - - client := http.Client{ - Timeout: 3 * time.Second, - } - resp, err := client.Do(req) - if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal("read response error") - } - // log.Println(string(body)) - response := msg.MultiResponse{} - err = json.Unmarshal(body, &response) - if err != nil { - log.Fatal("unmarshal resp error: ", err) - } - return response -} From 9de1f9d5cc021258d72f34e8b2d866c3181c0510 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 16 Apr 2022 03:56:17 +0200 Subject: [PATCH 02/35] remove packages --- pkg/histanal/histeval.go | 246 --------------------------- pkg/histanal/histload.go | 180 -------------------- pkg/strat/directory-sensitive.go | 47 ----- pkg/strat/dummy.go | 29 ---- pkg/strat/dynamic-record-distance.go | 91 ---------- pkg/strat/frequent.go | 53 ------ pkg/strat/markov-chain-cmd.go | 97 ----------- pkg/strat/markov-chain.go | 76 --------- pkg/strat/random.go | 57 ------- pkg/strat/recent-bash.go | 56 ------ pkg/strat/recent.go | 37 ---- pkg/strat/record-distance.go | 70 -------- pkg/strat/strat.go | 46 ----- 13 files changed, 1085 deletions(-) delete mode 100644 pkg/histanal/histeval.go delete mode 100644 pkg/histanal/histload.go delete mode 100644 pkg/strat/directory-sensitive.go delete mode 100644 pkg/strat/dummy.go delete mode 100644 pkg/strat/dynamic-record-distance.go delete mode 100644 pkg/strat/frequent.go delete mode 100644 pkg/strat/markov-chain-cmd.go delete mode 100644 pkg/strat/markov-chain.go delete mode 100644 pkg/strat/random.go delete mode 100644 pkg/strat/recent-bash.go delete mode 100644 pkg/strat/recent.go delete mode 100644 pkg/strat/record-distance.go delete mode 100644 pkg/strat/strat.go diff --git a/pkg/histanal/histeval.go b/pkg/histanal/histeval.go deleted file mode 100644 index 4d19779..0000000 --- a/pkg/histanal/histeval.go +++ /dev/null @@ -1,246 +0,0 @@ -package histanal - -import ( - "bytes" - "encoding/json" - "fmt" - "log" - "math/rand" - "os" - "os/exec" - - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/strat" - "github.com/jpillora/longestcommon" - - "github.com/schollz/progressbar" -) - -type matchJSON struct { - Match bool - Distance int - CharsRecalled int -} - -type multiMatchItemJSON struct { - Distance int - CharsRecalled int -} - -type multiMatchJSON struct { - Match bool - Entries []multiMatchItemJSON -} - -type strategyJSON struct { - Title string - Description string - Matches []matchJSON - PrefixMatches []multiMatchJSON -} - -// HistEval evaluates history -type HistEval struct { - HistLoad - BatchMode bool - maxCandidates int - Strategies []strategyJSON -} - -// NewHistEval constructs new HistEval -func NewHistEval(inputPath string, - maxCandidates int, skipFailedCmds bool, - debugRecords float64, sanitizedInput bool) HistEval { - - e := HistEval{ - HistLoad: HistLoad{ - skipFailedCmds: skipFailedCmds, - debugRecords: debugRecords, - sanitizedInput: sanitizedInput, - }, - maxCandidates: maxCandidates, - BatchMode: false, - } - records := e.loadHistoryRecords(inputPath) - device := deviceRecords{Records: records} - user := userRecords{} - user.Devices = append(user.Devices, device) - e.UsersRecords = append(e.UsersRecords, user) - e.preprocessRecords() - return e -} - -// NewHistEvalBatchMode constructs new HistEval in batch mode -func NewHistEvalBatchMode(input string, inputDataRoot string, - maxCandidates int, skipFailedCmds bool, - debugRecords float64, sanitizedInput bool) HistEval { - - e := HistEval{ - HistLoad: HistLoad{ - skipFailedCmds: skipFailedCmds, - debugRecords: debugRecords, - sanitizedInput: sanitizedInput, - }, - maxCandidates: maxCandidates, - BatchMode: false, - } - e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) - e.preprocessRecords() - return e -} - -func (e *HistEval) preprocessDeviceRecords(device deviceRecords) deviceRecords { - sessionIDs := map[string]uint64{} - var nextID uint64 - nextID = 1 // start with 1 because 0 won't get saved to json - for k, record := range device.Records { - id, found := sessionIDs[record.SessionID] - if found == false { - id = nextID - sessionIDs[record.SessionID] = id - nextID++ - } - device.Records[k].SeqSessionID = id - // assert - if record.Sanitized != e.sanitizedInput { - if e.sanitizedInput { - log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") - } - log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") - } - device.Records[k].SeqSessionID = id - if e.debugRecords > 0 && rand.Float64() < e.debugRecords { - device.Records[k].DebugThisRecord = true - } - } - // sort.SliceStable(device.Records, func(x, y int) bool { - // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { - // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal - // } - // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID - // }) - - // iterate from back and mark last record of each session - sessionIDSet := map[string]bool{} - for i := len(device.Records) - 1; i >= 0; i-- { - var record *records.EnrichedRecord - record = &device.Records[i] - if sessionIDSet[record.SessionID] { - continue - } - sessionIDSet[record.SessionID] = true - record.LastRecordOfSession = true - } - return device -} - -// enrich records and add sequential session ID -func (e *HistEval) preprocessRecords() { - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) - } - } -} - -// Evaluate a given strategy -func (e *HistEval) Evaluate(strategy strat.IStrategy) error { - title, description := strategy.GetTitleAndDescription() - log.Println("Evaluating strategy:", title, "-", description) - strategyData := strategyJSON{Title: title, Description: description} - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) - var prevRecord records.EnrichedRecord - for _, record := range e.UsersRecords[i].Devices[j].Records { - if e.skipFailedCmds && record.ExitCode != 0 { - continue - } - candidates := strategy.GetCandidates(records.Stripped(record)) - if record.DebugThisRecord { - log.Println() - log.Println("===================================================") - log.Println("STRATEGY:", title, "-", description) - log.Println("===================================================") - log.Println("Previous record:") - if prevRecord.RealtimeBefore == 0 { - log.Println("== NIL") - } else { - rec, _ := prevRecord.ToString() - log.Println(rec) - } - log.Println("---------------------------------------------------") - log.Println("Recommendations for:") - rec, _ := record.ToString() - log.Println(rec) - log.Println("---------------------------------------------------") - for i, candidate := range candidates { - if i > 10 { - break - } - log.Println(string(candidate)) - } - log.Println("===================================================") - } - - matchFound := false - longestPrefixMatchLength := 0 - multiMatch := multiMatchJSON{} - for i, candidate := range candidates { - // make an option (--calculate-total) to turn this on/off ? - // if i >= e.maxCandidates { - // break - // } - commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine})) - if commonPrefixLength > longestPrefixMatchLength { - longestPrefixMatchLength = commonPrefixLength - prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength} - multiMatch.Match = true - multiMatch.Entries = append(multiMatch.Entries, prefixMatch) - } - if candidate == record.CmdLine { - match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} - matchFound = true - strategyData.Matches = append(strategyData.Matches, match) - strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) - break - } - } - if matchFound == false { - strategyData.Matches = append(strategyData.Matches, matchJSON{}) - strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) - } - err := strategy.AddHistoryRecord(&record) - if err != nil { - log.Println("Error while evauating", err) - return err - } - bar.Add(1) - prevRecord = record - } - strategy.ResetHistory() - fmt.Println() - } - } - e.Strategies = append(e.Strategies, strategyData) - return nil -} - -// CalculateStatsAndPlot results -func (e *HistEval) CalculateStatsAndPlot(scriptName string) { - evalJSON, err := json.Marshal(e) - if err != nil { - log.Fatal("json marshal error", err) - } - buffer := bytes.Buffer{} - buffer.Write(evalJSON) - // run python script to stat and plot/ - cmd := exec.Command(scriptName) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = &buffer - err = cmd.Run() - if err != nil { - log.Printf("Command finished with error: %v", err) - } -} diff --git a/pkg/histanal/histload.go b/pkg/histanal/histload.go deleted file mode 100644 index ec81cc2..0000000 --- a/pkg/histanal/histload.go +++ /dev/null @@ -1,180 +0,0 @@ -package histanal - -import ( - "bufio" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "math/rand" - "os" - "path/filepath" - - "github.com/curusarn/resh/pkg/records" -) - -type deviceRecords struct { - Name string - Records []records.EnrichedRecord -} - -type userRecords struct { - Name string - Devices []deviceRecords -} - -// HistLoad loads history -type HistLoad struct { - UsersRecords []userRecords - skipFailedCmds bool - sanitizedInput bool - debugRecords float64 -} - -func (e *HistLoad) preprocessDeviceRecords(device deviceRecords) deviceRecords { - sessionIDs := map[string]uint64{} - var nextID uint64 - nextID = 1 // start with 1 because 0 won't get saved to json - for k, record := range device.Records { - id, found := sessionIDs[record.SessionID] - if found == false { - id = nextID - sessionIDs[record.SessionID] = id - nextID++ - } - device.Records[k].SeqSessionID = id - // assert - if record.Sanitized != e.sanitizedInput { - if e.sanitizedInput { - log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") - } - log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") - } - device.Records[k].SeqSessionID = id - if e.debugRecords > 0 && rand.Float64() < e.debugRecords { - device.Records[k].DebugThisRecord = true - } - } - // sort.SliceStable(device.Records, func(x, y int) bool { - // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { - // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal - // } - // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID - // }) - - // iterate from back and mark last record of each session - sessionIDSet := map[string]bool{} - for i := len(device.Records) - 1; i >= 0; i-- { - var record *records.EnrichedRecord - record = &device.Records[i] - if sessionIDSet[record.SessionID] { - continue - } - sessionIDSet[record.SessionID] = true - record.LastRecordOfSession = true - } - return device -} - -// enrich records and add sequential session ID -func (e *HistLoad) preprocessRecords() { - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) - } - } -} - -func (e *HistLoad) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords { - var records []userRecords - info, err := os.Stat(dataRootPath) - if err != nil { - log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")") - } - if info.IsDir() == false { - log.Fatal("Error:", dataRootPath, "is not a directory - exiting!") - } - users, err := ioutil.ReadDir(dataRootPath) - if err != nil { - log.Fatal("Could not read directory:", dataRootPath) - } - fmt.Println("Listing users in <", dataRootPath, ">...") - for _, user := range users { - userRecords := userRecords{Name: user.Name()} - userFullPath := filepath.Join(dataRootPath, user.Name()) - if user.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.") - continue - } - fmt.Println() - fmt.Printf("*- %s\n", user.Name()) - devices, err := ioutil.ReadDir(userFullPath) - if err != nil { - log.Fatal("Could not read directory:", userFullPath) - } - for _, device := range devices { - deviceRecords := deviceRecords{Name: device.Name()} - deviceFullPath := filepath.Join(userFullPath, device.Name()) - if device.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.") - continue - } - fmt.Printf(" \\- %s\n", device.Name()) - files, err := ioutil.ReadDir(deviceFullPath) - if err != nil { - log.Fatal("Could not read directory:", deviceFullPath) - } - for _, file := range files { - fileFullPath := filepath.Join(deviceFullPath, file.Name()) - if file.Name() == fname { - fmt.Printf(" \\- %s - loading ...", file.Name()) - // load the data - deviceRecords.Records = e.loadHistoryRecords(fileFullPath) - fmt.Println(" OK ✓") - } else { - fmt.Printf(" \\- %s - skipped\n", file.Name()) - } - } - userRecords.Devices = append(userRecords.Devices, deviceRecords) - } - records = append(records, userRecords) - } - return records -} - -func (e *HistLoad) loadHistoryRecords(fname string) []records.EnrichedRecord { - file, err := os.Open(fname) - if err != nil { - log.Fatal("Open() resh history file error:", err) - } - defer file.Close() - - var recs []records.EnrichedRecord - scanner := bufio.NewScanner(file) - for scanner.Scan() { - record := records.Record{} - fallbackRecord := records.FallbackRecord{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) - } - record = records.Convert(&fallbackRecord) - } - if e.sanitizedInput == false { - if record.CmdLength != 0 { - log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?") - } - record.CmdLength = len(record.CmdLine) - } else if record.CmdLength == 0 { - log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") - } - if !e.skipFailedCmds || record.ExitCode == 0 { - recs = append(recs, records.Enriched(record)) - } - } - return recs -} diff --git a/pkg/strat/directory-sensitive.go b/pkg/strat/directory-sensitive.go deleted file mode 100644 index 89d030e..0000000 --- a/pkg/strat/directory-sensitive.go +++ /dev/null @@ -1,47 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// DirectorySensitive prediction/recommendation strategy -type DirectorySensitive struct { - history map[string][]string - lastPwd string -} - -// Init see name -func (s *DirectorySensitive) Init() { - s.history = map[string][]string{} -} - -// GetTitleAndDescription see name -func (s *DirectorySensitive) GetTitleAndDescription() (string, string) { - return "directory sensitive (recent)", "Use recent commands executed is the same directory" -} - -// GetCandidates see name -func (s *DirectorySensitive) GetCandidates() []string { - return s.history[s.lastPwd] -} - -// AddHistoryRecord see name -func (s *DirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error { - // work on history for PWD - pwd := record.Pwd - // remove previous occurance of record - for i, cmd := range s.history[pwd] { - if cmd == record.CmdLine { - s.history[pwd] = append(s.history[pwd][:i], s.history[pwd][i+1:]...) - } - } - // append new record - s.history[pwd] = append([]string{record.CmdLine}, s.history[pwd]...) - s.lastPwd = record.PwdAfter - return nil -} - -// ResetHistory see name -func (s *DirectorySensitive) ResetHistory() error { - s.Init() - s.history = map[string][]string{} - return nil -} diff --git a/pkg/strat/dummy.go b/pkg/strat/dummy.go deleted file mode 100644 index fc813f2..0000000 --- a/pkg/strat/dummy.go +++ /dev/null @@ -1,29 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// Dummy prediction/recommendation strategy -type Dummy struct { - history []string -} - -// GetTitleAndDescription see name -func (s *Dummy) GetTitleAndDescription() (string, string) { - return "dummy", "Return empty candidate list" -} - -// GetCandidates see name -func (s *Dummy) GetCandidates() []string { - return nil -} - -// AddHistoryRecord see name -func (s *Dummy) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append(s.history, record.CmdLine) - return nil -} - -// ResetHistory see name -func (s *Dummy) ResetHistory() error { - return nil -} diff --git a/pkg/strat/dynamic-record-distance.go b/pkg/strat/dynamic-record-distance.go deleted file mode 100644 index 1f779c2..0000000 --- a/pkg/strat/dynamic-record-distance.go +++ /dev/null @@ -1,91 +0,0 @@ -package strat - -import ( - "math" - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" -) - -// DynamicRecordDistance prediction/recommendation strategy -type DynamicRecordDistance struct { - history []records.EnrichedRecord - DistParams records.DistParams - pwdHistogram map[string]int - realPwdHistogram map[string]int - gitOriginHistogram map[string]int - MaxDepth int - Label string -} - -type strDynDistEntry struct { - cmdLine string - distance float64 -} - -// Init see name -func (s *DynamicRecordDistance) Init() { - s.history = nil - s.pwdHistogram = map[string]int{} - s.realPwdHistogram = map[string]int{} - s.gitOriginHistogram = map[string]int{} -} - -// GetTitleAndDescription see name -func (s *DynamicRecordDistance) GetTitleAndDescription() (string, string) { - return "dynamic record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use TF-IDF record distance to recommend commands" -} - -func (s *DynamicRecordDistance) idf(count int) float64 { - return math.Log(float64(len(s.history)) / float64(count)) -} - -// GetCandidates see name -func (s *DynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { - if len(s.history) == 0 { - return nil - } - var mapItems []strDynDistEntry - for i, record := range s.history { - if s.MaxDepth != 0 && i > s.MaxDepth { - break - } - distParams := records.DistParams{ - Pwd: s.DistParams.Pwd * s.idf(s.pwdHistogram[strippedRecord.PwdAfter]), - RealPwd: s.DistParams.RealPwd * s.idf(s.realPwdHistogram[strippedRecord.RealPwdAfter]), - Git: s.DistParams.Git * s.idf(s.gitOriginHistogram[strippedRecord.GitOriginRemote]), - Time: s.DistParams.Time, - SessionID: s.DistParams.SessionID, - } - distance := record.DistanceTo(strippedRecord, distParams) - mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance}) - } - sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) - var hist []string - histSet := map[string]bool{} - for _, item := range mapItems { - if histSet[item.cmdLine] { - continue - } - histSet[item.cmdLine] = true - hist = append(hist, item.cmdLine) - } - return hist -} - -// AddHistoryRecord see name -func (s *DynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { - // append record to front - s.history = append([]records.EnrichedRecord{*record}, s.history...) - s.pwdHistogram[record.Pwd]++ - s.realPwdHistogram[record.RealPwd]++ - s.gitOriginHistogram[record.GitOriginRemote]++ - return nil -} - -// ResetHistory see name -func (s *DynamicRecordDistance) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/frequent.go b/pkg/strat/frequent.go deleted file mode 100644 index ff3b912..0000000 --- a/pkg/strat/frequent.go +++ /dev/null @@ -1,53 +0,0 @@ -package strat - -import ( - "sort" - - "github.com/curusarn/resh/pkg/records" -) - -// Frequent prediction/recommendation strategy -type Frequent struct { - history map[string]int -} - -type strFrqEntry struct { - cmdLine string - count int -} - -// Init see name -func (s *Frequent) Init() { - s.history = map[string]int{} -} - -// GetTitleAndDescription see name -func (s *Frequent) GetTitleAndDescription() (string, string) { - return "frequent", "Use frequent commands" -} - -// GetCandidates see name -func (s *Frequent) GetCandidates() []string { - var mapItems []strFrqEntry - for cmdLine, count := range s.history { - mapItems = append(mapItems, strFrqEntry{cmdLine, count}) - } - sort.Slice(mapItems, func(i int, j int) bool { return mapItems[i].count > mapItems[j].count }) - var hist []string - for _, item := range mapItems { - hist = append(hist, item.cmdLine) - } - return hist -} - -// AddHistoryRecord see name -func (s *Frequent) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history[record.CmdLine]++ - return nil -} - -// ResetHistory see name -func (s *Frequent) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/markov-chain-cmd.go b/pkg/strat/markov-chain-cmd.go deleted file mode 100644 index b1fa2f5..0000000 --- a/pkg/strat/markov-chain-cmd.go +++ /dev/null @@ -1,97 +0,0 @@ -package strat - -import ( - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" - "github.com/mb-14/gomarkov" -) - -// MarkovChainCmd prediction/recommendation strategy -type MarkovChainCmd struct { - Order int - history []strMarkCmdHistoryEntry - historyCmds []string -} - -type strMarkCmdHistoryEntry struct { - cmd string - cmdLine string -} - -type strMarkCmdEntry struct { - cmd string - transProb float64 -} - -// Init see name -func (s *MarkovChainCmd) Init() { - s.history = nil - s.historyCmds = nil -} - -// GetTitleAndDescription see name -func (s *MarkovChainCmd) GetTitleAndDescription() (string, string) { - return "command-based markov chain (order " + strconv.Itoa(s.Order) + ")", "Use command-based markov chain to recommend commands" -} - -// GetCandidates see name -func (s *MarkovChainCmd) GetCandidates() []string { - if len(s.history) < s.Order { - var hist []string - for _, item := range s.history { - hist = append(hist, item.cmdLine) - } - return hist - } - chain := gomarkov.NewChain(s.Order) - - chain.Add(s.historyCmds) - - cmdsSet := map[string]bool{} - var entries []strMarkCmdEntry - for _, cmd := range s.historyCmds { - if cmdsSet[cmd] { - continue - } - cmdsSet[cmd] = true - prob, _ := chain.TransitionProbability(cmd, s.historyCmds[len(s.historyCmds)-s.Order:]) - entries = append(entries, strMarkCmdEntry{cmd: cmd, transProb: prob}) - } - sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) - var hist []string - histSet := map[string]bool{} - for i := len(s.history) - 1; i >= 0; i-- { - if histSet[s.history[i].cmdLine] { - continue - } - histSet[s.history[i].cmdLine] = true - if s.history[i].cmd == entries[0].cmd { - hist = append(hist, s.history[i].cmdLine) - } - } - // log.Println("################") - // log.Println(s.history[len(s.history)-s.order:]) - // log.Println(" -> ") - // x := math.Min(float64(len(hist)), 3) - // log.Println(entries[:int(x)]) - // x = math.Min(float64(len(hist)), 5) - // log.Println(hist[:int(x)]) - // log.Println("################") - return hist -} - -// AddHistoryRecord see name -func (s *MarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command}) - s.historyCmds = append(s.historyCmds, record.Command) - // s.historySet[record.CmdLine] = true - return nil -} - -// ResetHistory see name -func (s *MarkovChainCmd) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/markov-chain.go b/pkg/strat/markov-chain.go deleted file mode 100644 index 50c7fdc..0000000 --- a/pkg/strat/markov-chain.go +++ /dev/null @@ -1,76 +0,0 @@ -package strat - -import ( - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" - "github.com/mb-14/gomarkov" -) - -// MarkovChain prediction/recommendation strategy -type MarkovChain struct { - Order int - history []string -} - -type strMarkEntry struct { - cmdLine string - transProb float64 -} - -// Init see name -func (s *MarkovChain) Init() { - s.history = nil -} - -// GetTitleAndDescription see name -func (s *MarkovChain) GetTitleAndDescription() (string, string) { - return "markov chain (order " + strconv.Itoa(s.Order) + ")", "Use markov chain to recommend commands" -} - -// GetCandidates see name -func (s *MarkovChain) GetCandidates() []string { - if len(s.history) < s.Order { - return s.history - } - chain := gomarkov.NewChain(s.Order) - - chain.Add(s.history) - - cmdLinesSet := map[string]bool{} - var entries []strMarkEntry - for _, cmdLine := range s.history { - if cmdLinesSet[cmdLine] { - continue - } - cmdLinesSet[cmdLine] = true - prob, _ := chain.TransitionProbability(cmdLine, s.history[len(s.history)-s.Order:]) - entries = append(entries, strMarkEntry{cmdLine: cmdLine, transProb: prob}) - } - sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) - var hist []string - for _, item := range entries { - hist = append(hist, item.cmdLine) - } - // log.Println("################") - // log.Println(s.history[len(s.history)-s.order:]) - // log.Println(" -> ") - // x := math.Min(float64(len(hist)), 5) - // log.Println(hist[:int(x)]) - // log.Println("################") - return hist -} - -// AddHistoryRecord see name -func (s *MarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append(s.history, record.CmdLine) - // s.historySet[record.CmdLine] = true - return nil -} - -// ResetHistory see name -func (s *MarkovChain) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/random.go b/pkg/strat/random.go deleted file mode 100644 index 0ff52f1..0000000 --- a/pkg/strat/random.go +++ /dev/null @@ -1,57 +0,0 @@ -package strat - -import ( - "math/rand" - "time" - - "github.com/curusarn/resh/pkg/records" -) - -// Random prediction/recommendation strategy -type Random struct { - CandidatesSize int - history []string - historySet map[string]bool -} - -// Init see name -func (s *Random) Init() { - s.history = nil - s.historySet = map[string]bool{} -} - -// GetTitleAndDescription see name -func (s *Random) GetTitleAndDescription() (string, string) { - return "random", "Use random commands" -} - -// GetCandidates see name -func (s *Random) GetCandidates() []string { - seed := time.Now().UnixNano() - rand.Seed(seed) - var candidates []string - candidateSet := map[string]bool{} - for len(candidates) < s.CandidatesSize && len(candidates)*2 < len(s.historySet) { - x := rand.Intn(len(s.history)) - candidate := s.history[x] - if candidateSet[candidate] == false { - candidateSet[candidate] = true - candidates = append(candidates, candidate) - continue - } - } - return candidates -} - -// AddHistoryRecord see name -func (s *Random) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append([]string{record.CmdLine}, s.history...) - s.historySet[record.CmdLine] = true - return nil -} - -// ResetHistory see name -func (s *Random) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/recent-bash.go b/pkg/strat/recent-bash.go deleted file mode 100644 index ace3571..0000000 --- a/pkg/strat/recent-bash.go +++ /dev/null @@ -1,56 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// RecentBash prediction/recommendation strategy -type RecentBash struct { - histfile []string - histfileSnapshot map[string][]string - history map[string][]string -} - -// Init see name -func (s *RecentBash) Init() { - s.histfileSnapshot = map[string][]string{} - s.history = map[string][]string{} -} - -// GetTitleAndDescription see name -func (s *RecentBash) GetTitleAndDescription() (string, string) { - return "recent (bash-like)", "Behave like bash" -} - -// GetCandidates see name -func (s *RecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string { - // populate the local history from histfile - if s.histfileSnapshot[strippedRecord.SessionID] == nil { - s.histfileSnapshot[strippedRecord.SessionID] = s.histfile - } - return append(s.history[strippedRecord.SessionID], s.histfileSnapshot[strippedRecord.SessionID]...) -} - -// AddHistoryRecord see name -func (s *RecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { - // remove previous occurance of record - for i, cmd := range s.history[record.SessionID] { - if cmd == record.CmdLine { - s.history[record.SessionID] = append(s.history[record.SessionID][:i], s.history[record.SessionID][i+1:]...) - } - } - // append new record - s.history[record.SessionID] = append([]string{record.CmdLine}, s.history[record.SessionID]...) - - if record.LastRecordOfSession { - // append history of the session to histfile and clear session history - s.histfile = append(s.history[record.SessionID], s.histfile...) - s.histfileSnapshot[record.SessionID] = nil - s.history[record.SessionID] = nil - } - return nil -} - -// ResetHistory see name -func (s *RecentBash) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/recent.go b/pkg/strat/recent.go deleted file mode 100644 index 157b52c..0000000 --- a/pkg/strat/recent.go +++ /dev/null @@ -1,37 +0,0 @@ -package strat - -import "github.com/curusarn/resh/pkg/records" - -// Recent prediction/recommendation strategy -type Recent struct { - history []string -} - -// GetTitleAndDescription see name -func (s *Recent) GetTitleAndDescription() (string, string) { - return "recent", "Use recent commands" -} - -// GetCandidates see name -func (s *Recent) GetCandidates() []string { - return s.history -} - -// AddHistoryRecord see name -func (s *Recent) AddHistoryRecord(record *records.EnrichedRecord) error { - // remove previous occurance of record - for i, cmd := range s.history { - if cmd == record.CmdLine { - s.history = append(s.history[:i], s.history[i+1:]...) - } - } - // append new record - s.history = append([]string{record.CmdLine}, s.history...) - return nil -} - -// ResetHistory see name -func (s *Recent) ResetHistory() error { - s.history = nil - return nil -} diff --git a/pkg/strat/record-distance.go b/pkg/strat/record-distance.go deleted file mode 100644 index e582584..0000000 --- a/pkg/strat/record-distance.go +++ /dev/null @@ -1,70 +0,0 @@ -package strat - -import ( - "sort" - "strconv" - - "github.com/curusarn/resh/pkg/records" -) - -// RecordDistance prediction/recommendation strategy -type RecordDistance struct { - history []records.EnrichedRecord - DistParams records.DistParams - MaxDepth int - Label string -} - -type strDistEntry struct { - cmdLine string - distance float64 -} - -// Init see name -func (s *RecordDistance) Init() { - s.history = nil -} - -// GetTitleAndDescription see name -func (s *RecordDistance) GetTitleAndDescription() (string, string) { - return "record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use record distance to recommend commands" -} - -// GetCandidates see name -func (s *RecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { - if len(s.history) == 0 { - return nil - } - var mapItems []strDistEntry - for i, record := range s.history { - if s.MaxDepth != 0 && i > s.MaxDepth { - break - } - distance := record.DistanceTo(strippedRecord, s.DistParams) - mapItems = append(mapItems, strDistEntry{record.CmdLine, distance}) - } - sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) - var hist []string - histSet := map[string]bool{} - for _, item := range mapItems { - if histSet[item.cmdLine] { - continue - } - histSet[item.cmdLine] = true - hist = append(hist, item.cmdLine) - } - return hist -} - -// AddHistoryRecord see name -func (s *RecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { - // append record to front - s.history = append([]records.EnrichedRecord{*record}, s.history...) - return nil -} - -// ResetHistory see name -func (s *RecordDistance) ResetHistory() error { - s.Init() - return nil -} diff --git a/pkg/strat/strat.go b/pkg/strat/strat.go deleted file mode 100644 index 28ac015..0000000 --- a/pkg/strat/strat.go +++ /dev/null @@ -1,46 +0,0 @@ -package strat - -import ( - "github.com/curusarn/resh/pkg/records" -) - -// ISimpleStrategy interface -type ISimpleStrategy interface { - GetTitleAndDescription() (string, string) - GetCandidates() []string - AddHistoryRecord(record *records.EnrichedRecord) error - ResetHistory() error -} - -// IStrategy interface -type IStrategy interface { - GetTitleAndDescription() (string, string) - GetCandidates(r records.EnrichedRecord) []string - AddHistoryRecord(record *records.EnrichedRecord) error - ResetHistory() error -} - -type simpleStrategyWrapper struct { - strategy ISimpleStrategy -} - -// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy -func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper { - return &simpleStrategyWrapper{strategy: strategy} -} - -func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) { - return s.strategy.GetTitleAndDescription() -} - -func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string { - return s.strategy.GetCandidates() -} - -func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error { - return s.strategy.AddHistoryRecord(r) -} - -func (s *simpleStrategyWrapper) ResetHistory() error { - return s.strategy.ResetHistory() -} From 40be2eb87ee394bdcbeaf382daaa7a8137025da3 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 16 Apr 2022 04:00:23 +0200 Subject: [PATCH 03/35] remove /recall and /inspect --- cmd/daemon/run-server.go | 8 -- pkg/sesshist/sesshist.go | 243 --------------------------------------- 2 files changed, 251 deletions(-) delete mode 100644 pkg/sesshist/sesshist.go diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index 9114b20..0207b86 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -43,11 +43,6 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa maxHistSize, minHistSizeKB, histfileSignals, shutdown) - // sesshist New - sesshistDispatch := sesshist.NewDispatch(sesshistSessionsToInit, sesshistSessionsToDrop, - sesshistRecords, histfileBox, - config.SesshistInitHistorySize) - // sesswatch sesswatchRecords := make(chan records.Record) recordSubscribers = append(recordSubscribers, sesswatchRecords) @@ -60,9 +55,6 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa mux.HandleFunc("/status", statusHandler) mux.Handle("/record", &recordHandler{subscribers: recordSubscribers}) mux.Handle("/session_init", &sessionInitHandler{subscribers: sessionInitSubscribers}) - // TOOD: drop recall and inspect - mux.Handle("/recall", &recallHandler{sesshistDispatch: sesshistDispatch}) - mux.Handle("/inspect", &inspectHandler{sesshistDispatch: sesshistDispatch}) mux.Handle("/dump", &dumpHandler{histfileBox: histfileBox}) server := &http.Server{ diff --git a/pkg/sesshist/sesshist.go b/pkg/sesshist/sesshist.go deleted file mode 100644 index f722764..0000000 --- a/pkg/sesshist/sesshist.go +++ /dev/null @@ -1,243 +0,0 @@ -package sesshist - -import ( - "errors" - "log" - "strconv" - "strings" - "sync" - - "github.com/curusarn/resh/pkg/histfile" - "github.com/curusarn/resh/pkg/histlist" - "github.com/curusarn/resh/pkg/records" -) - -// Dispatch Recall() calls to an apropriate session history (sesshist) -type Dispatch struct { - sessions map[string]*sesshist - mutex sync.RWMutex - - history *histfile.Histfile - historyInitSize int -} - -// NewDispatch creates a new sesshist.Dispatch and starts necessary gorutines -func NewDispatch(sessionsToInit chan records.Record, sessionsToDrop chan string, - recordsToAdd chan records.Record, history *histfile.Histfile, historyInitSize int) *Dispatch { - - s := Dispatch{ - sessions: map[string]*sesshist{}, - history: history, - historyInitSize: historyInitSize, - } - go s.sessionInitializer(sessionsToInit) - go s.sessionDropper(sessionsToDrop) - go s.recordAdder(recordsToAdd) - return &s -} - -func (s *Dispatch) sessionInitializer(sessionsToInit chan records.Record) { - for { - record := <-sessionsToInit - log.Println("sesshist: got session to init - " + record.SessionID) - s.initSession(record.SessionID, record.Shell) - } -} - -func (s *Dispatch) sessionDropper(sessionsToDrop chan string) { - for { - sessionID := <-sessionsToDrop - log.Println("sesshist: got session to drop - " + sessionID) - s.dropSession(sessionID) - } -} - -func (s *Dispatch) recordAdder(recordsToAdd chan records.Record) { - for { - record := <-recordsToAdd - if record.PartOne { - log.Println("sesshist: got record to add - " + record.CmdLine) - s.addRecentRecord(record.SessionID, record) - } else { - // this inits session on RESH update - s.checkSession(record.SessionID, record.Shell) - } - // TODO: we will need to handle part2 as well eventually - } -} - -func (s *Dispatch) checkSession(sessionID, shell string) { - s.mutex.RLock() - _, found := s.sessions[sessionID] - s.mutex.RUnlock() - if found == false { - err := s.initSession(sessionID, shell) - if err != nil { - log.Println("sesshist: Error while checking session:", err) - } - } -} - -// InitSession struct -func (s *Dispatch) initSession(sessionID, shell string) error { - log.Println("sesshist: initializing session - " + sessionID) - s.mutex.RLock() - _, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == true { - return errors.New("sesshist ERROR: Can't INIT already existing session " + sessionID) - } - - log.Println("sesshist: loading history to populate session - " + sessionID) - historyCmdLines := s.history.GetRecentCmdLines(shell, s.historyInitSize) - - s.mutex.Lock() - defer s.mutex.Unlock() - // init sesshist and populate it with history loaded from file - s.sessions[sessionID] = &sesshist{ - recentCmdLines: historyCmdLines, - } - log.Println("sesshist: session init done - " + sessionID) - return nil -} - -// DropSession struct -func (s *Dispatch) dropSession(sessionID string) error { - s.mutex.RLock() - _, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == false { - return errors.New("sesshist ERROR: Can't DROP not existing session " + sessionID) - } - - s.mutex.Lock() - defer s.mutex.Unlock() - delete(s.sessions, sessionID) - return nil -} - -// AddRecent record to session -func (s *Dispatch) addRecentRecord(sessionID string, record records.Record) error { - log.Println("sesshist: Adding a record, RLocking main lock ...") - s.mutex.RLock() - log.Println("sesshist: Getting a session ...") - session, found := s.sessions[sessionID] - log.Println("sesshist: RUnlocking main lock ...") - s.mutex.RUnlock() - - if found == false { - log.Println("sesshist ERROR: addRecentRecord(): No session history for SessionID " + sessionID + " - creating session history.") - s.initSession(sessionID, record.Shell) - return s.addRecentRecord(sessionID, record) - } - log.Println("sesshist: RLocking session lock (w/ defer) ...") - session.mutex.Lock() - defer session.mutex.Unlock() - session.recentRecords = append(session.recentRecords, record) - session.recentCmdLines.AddCmdLine(record.CmdLine) - log.Println("sesshist: record:", record.CmdLine, "; added to session:", sessionID, - "; session len:", len(session.recentCmdLines.List), "; session len (records):", len(session.recentRecords)) - return nil -} - -// Recall command from recent session history -func (s *Dispatch) Recall(sessionID string, histno int, prefix string) (string, error) { - log.Println("sesshist - recall: RLocking main lock ...") - s.mutex.RLock() - log.Println("sesshist - recall: Getting session history struct ...") - session, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == false { - // TODO: propagate actual shell here so we can use it - go s.initSession(sessionID, "bash") - return "", errors.New("sesshist ERROR: No session history for SessionID " + sessionID + " - creating one ...") - } - log.Println("sesshist - recall: Locking session lock ...") - session.mutex.Lock() - defer session.mutex.Unlock() - if prefix == "" { - log.Println("sesshist - recall: Getting records by histno ...") - return session.getRecordByHistno(histno) - } - log.Println("sesshist - recall: Searching for records by prefix ...") - return session.searchRecordByPrefix(prefix, histno) -} - -// Inspect commands in recent session history -func (s *Dispatch) Inspect(sessionID string, count int) ([]string, error) { - prefix := "" - log.Println("sesshist - inspect: RLocking main lock ...") - s.mutex.RLock() - log.Println("sesshist - inspect: Getting session history struct ...") - session, found := s.sessions[sessionID] - s.mutex.RUnlock() - - if found == false { - // go s.initSession(sessionID) - return nil, errors.New("sesshist ERROR: No session history for SessionID " + sessionID + " - should we create one?") - } - log.Println("sesshist - inspect: Locking session lock ...") - session.mutex.Lock() - defer session.mutex.Unlock() - if prefix == "" { - log.Println("sesshist - inspect: Getting records by histno ...") - idx := len(session.recentCmdLines.List) - count - if idx < 0 { - idx = 0 - } - return session.recentCmdLines.List[idx:], nil - } - log.Println("sesshist - inspect: Searching for records by prefix ... ERROR - Not implemented") - return nil, errors.New("sesshist ERROR: Inspect - Searching for records by prefix Not implemented yet") -} - -type sesshist struct { - mutex sync.Mutex - recentRecords []records.Record - recentCmdLines histlist.Histlist -} - -func (s *sesshist) getRecordByHistno(histno int) (string, error) { - // addRecords() appends records to the end of the slice - // -> this func handles the indexing - if histno == 0 { - return "", errors.New("sesshist ERROR: 'histno == 0' is not a record from history") - } - if histno < 0 { - return "", errors.New("sesshist ERROR: 'histno < 0' is a command from future (not supperted yet)") - } - index := len(s.recentCmdLines.List) - histno - if index < 0 { - return "", errors.New("sesshist ERROR: 'histno > number of commands in the session' (" + strconv.Itoa(len(s.recentCmdLines.List)) + ")") - } - return s.recentCmdLines.List[index], nil -} - -func (s *sesshist) searchRecordByPrefix(prefix string, histno int) (string, error) { - if histno == 0 { - return "", errors.New("sesshist ERROR: 'histno == 0' is not a record from history") - } - if histno < 0 { - return "", errors.New("sesshist ERROR: 'histno < 0' is a command from future (not supperted yet)") - } - index := len(s.recentCmdLines.List) - histno - if index < 0 { - return "", errors.New("sesshist ERROR: 'histno > number of commands in the session' (" + strconv.Itoa(len(s.recentCmdLines.List)) + ")") - } - cmdLines := []string{} - for i := len(s.recentCmdLines.List) - 1; i >= 0; i-- { - if strings.HasPrefix(s.recentCmdLines.List[i], prefix) { - cmdLines = append(cmdLines, s.recentCmdLines.List[i]) - if len(cmdLines) >= histno { - break - } - } - } - if len(cmdLines) < histno { - return "", errors.New("sesshist ERROR: 'histno > number of commands matching with given prefix' (" + strconv.Itoa(len(cmdLines)) + ")") - } - return cmdLines[histno-1], nil -} From 3937707aa4defa21e148927ed71d2f2ee1dc520a Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 16 Apr 2022 04:06:37 +0200 Subject: [PATCH 04/35] remove import --- cmd/daemon/recall.go | 109 --------------------------------------- cmd/daemon/run-server.go | 1 - 2 files changed, 110 deletions(-) delete mode 100644 cmd/daemon/recall.go diff --git a/cmd/daemon/recall.go b/cmd/daemon/recall.go deleted file mode 100644 index 930537c..0000000 --- a/cmd/daemon/recall.go +++ /dev/null @@ -1,109 +0,0 @@ -package main - -import ( - "encoding/json" - "io/ioutil" - "log" - "net/http" - - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/msg" - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/sesshist" -) - -type recallHandler struct { - sesshistDispatch *sesshist.Dispatch -} - -func (h *recallHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if Debug { - log.Println("/recall START") - log.Println("/recall reading body ...") - } - jsn, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Println("Error reading the body", err) - return - } - - rec := records.SlimRecord{} - if Debug { - log.Println("/recall unmarshaling record ...") - } - err = json.Unmarshal(jsn, &rec) - if err != nil { - log.Println("Decoding error:", err) - log.Println("Payload:", jsn) - return - } - if Debug { - log.Println("/recall recalling ...") - } - found := true - cmd, err := h.sesshistDispatch.Recall(rec.SessionID, rec.RecallHistno, rec.RecallPrefix) - if err != nil { - log.Println("/recall - sess id:", rec.SessionID, " - histno:", rec.RecallHistno, " -> ERROR") - log.Println("Recall error:", err) - found = false - cmd = "" - } - resp := collect.SingleResponse{CmdLine: cmd, Found: found} - if Debug { - log.Println("/recall marshaling response ...") - } - jsn, err = json.Marshal(&resp) - if err != nil { - log.Println("Encoding error:", err) - log.Println("Response:", resp) - return - } - if Debug { - log.Println(string(jsn)) - log.Println("/recall writing response ...") - } - w.Write(jsn) - log.Println("/recall END - sess id:", rec.SessionID, " - histno:", rec.RecallHistno, " -> ", cmd, " (found:", found, ")") -} - -type inspectHandler struct { - sesshistDispatch *sesshist.Dispatch -} - -func (h *inspectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - log.Println("/inspect START") - log.Println("/inspect reading body ...") - jsn, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Println("Error reading the body", err) - return - } - - mess := msg.InspectMsg{} - log.Println("/inspect unmarshaling record ...") - err = json.Unmarshal(jsn, &mess) - if err != nil { - log.Println("Decoding error:", err) - log.Println("Payload:", jsn) - return - } - log.Println("/inspect recalling ...") - cmds, err := h.sesshistDispatch.Inspect(mess.SessionID, int(mess.Count)) - if err != nil { - log.Println("/inspect - sess id:", mess.SessionID, " - count:", mess.Count, " -> ERROR") - log.Println("Inspect error:", err) - return - } - resp := msg.MultiResponse{CmdLines: cmds} - log.Println("/inspect marshaling response ...") - jsn, err = json.Marshal(&resp) - if err != nil { - log.Println("Encoding error:", err) - log.Println("Response:", resp) - return - } - // log.Println(string(jsn)) - log.Println("/inspect writing response ...") - w.Write(jsn) - log.Println("/inspect END - sess id:", mess.SessionID, " - count:", mess.Count) -} diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index 0207b86..fdad2bc 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -8,7 +8,6 @@ import ( "github.com/curusarn/resh/pkg/cfg" "github.com/curusarn/resh/pkg/histfile" "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/sesshist" "github.com/curusarn/resh/pkg/sesswatch" "github.com/curusarn/resh/pkg/signalhandler" ) From 5c576978508730405ac5661ae88794b86444fde4 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 16 Apr 2022 18:37:33 +0200 Subject: [PATCH 05/35] remove sesshist from run-server --- cmd/daemon/run-server.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index fdad2bc..9ca92a6 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -20,14 +20,6 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa shutdown := make(chan string) - // sessshist - sesshistSessionsToInit := make(chan records.Record) - sessionInitSubscribers = append(sessionInitSubscribers, sesshistSessionsToInit) - sesshistSessionsToDrop := make(chan string) - sessionDropSubscribers = append(sessionDropSubscribers, sesshistSessionsToDrop) - sesshistRecords := make(chan records.Record) - recordSubscribers = append(recordSubscribers, sesshistRecords) - // histfile histfileRecords := make(chan records.Record) recordSubscribers = append(recordSubscribers, histfileRecords) From d72fa1d2006d51dd486770c0fe84bb90153d8551 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Mon, 18 Apr 2022 23:55:24 +0200 Subject: [PATCH 06/35] remove recall --- .goreleaser.yml | 36 ---------- Makefile | 2 +- cmd/collect/main.go | 151 ++++++++++++++++------------------------- pkg/collect/collect.go | 35 ---------- roadmap.md | 2 - 5 files changed, 60 insertions(+), 166 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 11c048d..125b906 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -75,33 +75,6 @@ builds: - amd64 - arm - arm64 - - - id: "evaluate" - main: ./cmd/evaluate - binary: bin/resh-evaluate - goarch: - - 386 - - amd64 - - arm - - arm64 - - - id: "event" - main: ./cmd/event - binary: bin/resh-event - goarch: - - 386 - - amd64 - - arm - - arm64 - - - id: "inspect" - main: ./cmd/inspect - binary: bin/resh-inspect - goarch: - - 386 - - amd64 - - arm - - arm64 - id: "postcollect" main: ./cmd/postcollect @@ -111,15 +84,6 @@ builds: - amd64 - arm - arm64 - - - id: "sanitize" - main: ./cmd/sanitize - binary: bin/resh-sanitize - goarch: - - 386 - - amd64 - - arm - - arm64 - id: "session-init" main: ./cmd/session-init diff --git a/Makefile b/Makefile index 02bb7e0..251527b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${REVISION}" build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect bin/resh-daemon\ - bin/resh-evaluate bin/resh-sanitize bin/resh-control bin/resh-config bin/resh-inspect bin/resh-cli + bin/resh-control bin/resh-config bin/resh-cli install: build scripts/install.sh diff --git a/cmd/collect/main.go b/cmd/collect/main.go index d2d2815..57e6135 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -35,10 +35,6 @@ func main() { if _, err := toml.DecodeFile(configPath, &config); err != nil { log.Fatal("Error reading config:", err) } - // recall command - recall := flag.Bool("recall", false, "Recall command on position --histno") - recallHistno := flag.Int("histno", 0, "Recall command on position --histno") - recallPrefix := flag.String("prefix-search", "", "Recall command based on prefix --prefix-search") // version showVersion := flag.Bool("version", false, "Show version and exit") @@ -55,11 +51,6 @@ func main() { sessionID := flag.String("sessionId", "", "resh generated session id") recordID := flag.String("recordId", "", "resh generated record id") - // recall metadata - recallActions := flag.String("recall-actions", "", "recall actions that took place before executing the command") - recallStrategy := flag.String("recall-strategy", "", "recall strategy used during recall actions") - recallLastCmdLine := flag.String("recall-last-cmdline", "", "last recalled cmdline") - // posix variables cols := flag.String("cols", "-1", "$COLUMNS") lines := flag.String("lines", "-1", "$LINES") @@ -127,10 +118,6 @@ func main() { ")") os.Exit(3) } - if *recallPrefix != "" && *recall == false { - log.Println("Option '--prefix-search' only works with '--recall' option - exiting!") - os.Exit(4) - } realtimeBefore, err := strconv.ParseFloat(*rtb, 64) if err != nil { @@ -171,85 +158,65 @@ func main() { // *osReleasePrettyName = "Linux" // } - if *recall { - rec := records.SlimRecord{ - SessionID: *sessionID, - RecallHistno: *recallHistno, - RecallPrefix: *recallPrefix, - } - str, found := collect.SendRecallRequest(rec, strconv.Itoa(config.Port)) - if found == false { - os.Exit(1) - } - fmt.Println(str) - } else { - rec := records.Record{ + rec := records.Record{ + // posix + Cols: *cols, + Lines: *lines, + // core + BaseRecord: records.BaseRecord{ + CmdLine: *cmdLine, + ExitCode: *exitCode, + Shell: *shell, + Uname: *uname, + SessionID: *sessionID, + RecordID: *recordID, + // posix - Cols: *cols, - Lines: *lines, - // core - BaseRecord: records.BaseRecord{ - RecallHistno: *recallHistno, - - CmdLine: *cmdLine, - ExitCode: *exitCode, - Shell: *shell, - Uname: *uname, - SessionID: *sessionID, - RecordID: *recordID, - - // posix - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - Pwd: *pwd, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - RealPwd: realPwd, - Pid: *pid, - SessionPID: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - - RealtimeBefore: realtimeBefore, - RealtimeBeforeLocal: realtimeBeforeLocal, - - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - GitDir: gitDir, - GitRealDir: gitRealDir, - GitOriginRemote: *gitRemote, - MachineID: collect.ReadFileContent(machineIDPath), - - OsReleaseID: *osReleaseID, - OsReleaseVersionID: *osReleaseVersionID, - OsReleaseIDLike: *osReleaseIDLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, - - PartOne: true, - - ReshUUID: collect.ReadFileContent(reshUUIDPath), - ReshVersion: version, - ReshRevision: commit, - - RecallActionsRaw: *recallActions, - RecallPrefix: *recallPrefix, - RecallStrategy: *recallStrategy, - RecallLastCmdLine: *recallLastCmdLine, - }, - } - collect.SendRecord(rec, strconv.Itoa(config.Port), "/record") + Home: *home, + Lang: *lang, + LcAll: *lcAll, + Login: *login, + // Path: *path, + Pwd: *pwd, + ShellEnv: *shellEnv, + Term: *term, + + // non-posix + RealPwd: realPwd, + Pid: *pid, + SessionPID: *sessionPid, + Host: *host, + Hosttype: *hosttype, + Ostype: *ostype, + Machtype: *machtype, + Shlvl: *shlvl, + + // before after + TimezoneBefore: *timezoneBefore, + + RealtimeBefore: realtimeBefore, + RealtimeBeforeLocal: realtimeBeforeLocal, + + RealtimeSinceSessionStart: realtimeSinceSessionStart, + RealtimeSinceBoot: realtimeSinceBoot, + + GitDir: gitDir, + GitRealDir: gitRealDir, + GitOriginRemote: *gitRemote, + MachineID: collect.ReadFileContent(machineIDPath), + + OsReleaseID: *osReleaseID, + OsReleaseVersionID: *osReleaseVersionID, + OsReleaseIDLike: *osReleaseIDLike, + OsReleaseName: *osReleaseName, + OsReleasePrettyName: *osReleasePrettyName, + + PartOne: true, + + ReshUUID: collect.ReadFileContent(reshUUIDPath), + ReshVersion: version, + ReshRevision: commit, + }, } + collect.SendRecord(rec, strconv.Itoa(config.Port), "/record") } diff --git a/pkg/collect/collect.go b/pkg/collect/collect.go index 5fd849b..b1c2fb6 100644 --- a/pkg/collect/collect.go +++ b/pkg/collect/collect.go @@ -20,41 +20,6 @@ type SingleResponse struct { CmdLine string `json:"cmdline"` } -// SendRecallRequest to daemon -func SendRecallRequest(r records.SlimRecord, port string) (string, bool) { - recJSON, err := json.Marshal(r) - if err != nil { - log.Fatal("send err 1", err) - } - - req, err := http.NewRequest("POST", "http://localhost:"+port+"/recall", - bytes.NewBuffer(recJSON)) - if err != nil { - log.Fatal("send err 2", err) - } - req.Header.Set("Content-Type", "application/json") - - client := httpclient.New() - resp, err := client.Do(req) - if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") - } - - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal("read response error") - } - log.Println(string(body)) - response := SingleResponse{} - err = json.Unmarshal(body, &response) - if err != nil { - log.Fatal("unmarshal resp error: ", err) - } - log.Println(response) - return response.CmdLine, response.Found -} - // SendRecord to daemon func SendRecord(r records.Record, port, path string) { recJSON, err := json.Marshal(r) diff --git a/roadmap.md b/roadmap.md index efcc2d5..e2be272 100644 --- a/roadmap.md +++ b/roadmap.md @@ -41,5 +41,3 @@ - :heavy_check_mark: Linux - :white_check_mark: MacOS *(requires coreutils - `brew install coreutils`)* -- :heavy_check_mark: Provide a tool to sanitize the recorded history - From c33db41180c59259c7bd613dce823ba6507bc779 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 19 Apr 2022 00:01:56 +0200 Subject: [PATCH 07/35] remove --args --- scripts/hooks.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 02e74c3..5a194c1 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -18,9 +18,6 @@ __resh_preexec() { __RESH_CMDLINE="$1" # not local to preserve it for postcollect (useful as sanity check) local fpath_last_run="$__RESH_XDG_CACHE_HOME/collect_last_run_out.txt" __resh_collect --cmdLine "$__RESH_CMDLINE" \ - --recall-actions "$__RESH_HIST_RECALL_ACTIONS" \ - --recall-strategy "$__RESH_HIST_RECALL_STRATEGY" \ - --recall-last-cmdline "$__RESH_HIST_PREV_LINE" \ >| "$fpath_last_run" 2>&1 || echo "resh-collect ERROR: $(head -n 1 $fpath_last_run)" } @@ -113,7 +110,6 @@ __resh_collect() { -osReleaseIdLike "$__RESH_OS_RELEASE_ID_LIKE" \ -osReleaseName "$__RESH_OS_RELEASE_NAME" \ -osReleasePrettyName "$__RESH_OS_RELEASE_PRETTY_NAME" \ - -histno "$__RESH_HISTNO" \ "$@" return $? fi From f033a2190286664445e730493271d7559fe94525 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 19 Apr 2022 00:10:43 +0200 Subject: [PATCH 08/35] todo --- scripts/hooks.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 5a194c1..b4138ef 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -1,5 +1,6 @@ __resh_reset_variables() { + # TODO: remove this block __RESH_HISTNO=0 __RESH_HISTNO_MAX="" __RESH_HISTNO_ZERO_LINE="" @@ -9,6 +10,7 @@ __resh_reset_variables() { __RESH_HIST_RECALL_ACTIONS="" __RESH_HIST_NO_PREFIX_MODE=0 __RESH_HIST_RECALL_STRATEGY="" + # end TODO __RESH_RECORD_ID=$(__resh_get_uuid) } From 0788ca9005d34553f4e0d124e5a6af51dd6fa7ab Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 19 Apr 2022 22:49:13 +0200 Subject: [PATCH 09/35] remove deprecated variables --- scripts/hooks.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/scripts/hooks.sh b/scripts/hooks.sh index b4138ef..25e7e02 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -1,16 +1,5 @@ __resh_reset_variables() { - # TODO: remove this block - __RESH_HISTNO=0 - __RESH_HISTNO_MAX="" - __RESH_HISTNO_ZERO_LINE="" - __RESH_HIST_PREV_LINE="" - __RESH_HIST_PREV_CURSOR="" # deprecated - __RESH_HIST_PREV_PREFIX="" - __RESH_HIST_RECALL_ACTIONS="" - __RESH_HIST_NO_PREFIX_MODE=0 - __RESH_HIST_RECALL_STRATEGY="" - # end TODO __RESH_RECORD_ID=$(__resh_get_uuid) } From 0fcad6310c41050f553021cd235e70e360015d1b Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 20 Apr 2022 01:23:59 +0200 Subject: [PATCH 10/35] fix double subscribe of sessWatch, minor fixes --- cmd/daemon/record.go | 6 +++--- cmd/daemon/run-server.go | 2 +- cmd/daemon/session-init.go | 2 +- pkg/sesswatch/sesswatch.go | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/daemon/record.go b/cmd/daemon/record.go index ee403f8..38dce4b 100644 --- a/cmd/daemon/record.go +++ b/cmd/daemon/record.go @@ -30,14 +30,14 @@ func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.Println("Payload: ", jsn) return } - for _, sub := range h.subscribers { - sub <- record - } part := "2" if record.PartOne { part = "1" } log.Println("/record - ", record.CmdLine, " - part", part) + for _, sub := range h.subscribers { + sub <- record + } }() // fmt.Println("cmd:", r.CmdLine) diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index 9ca92a6..9ad1310 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -38,7 +38,7 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa sesswatchRecords := make(chan records.Record) recordSubscribers = append(recordSubscribers, sesswatchRecords) sesswatchSessionsToWatch := make(chan records.Record) - sessionInitSubscribers = append(sessionInitSubscribers, sesswatchRecords, sesswatchSessionsToWatch) + sessionInitSubscribers = append(sessionInitSubscribers, sesswatchSessionsToWatch) sesswatch.Go(sesswatchSessionsToWatch, sesswatchRecords, sessionDropSubscribers, config.SesswatchPeriodSeconds) // handlers diff --git a/cmd/daemon/session-init.go b/cmd/daemon/session-init.go index 27a1b27..da1e236 100644 --- a/cmd/daemon/session-init.go +++ b/cmd/daemon/session-init.go @@ -30,9 +30,9 @@ func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.Println("Payload: ", jsn) return } + log.Println("/session_init - id:", record.SessionID, " - pid:", record.SessionPID) for _, sub := range h.subscribers { sub <- record } - log.Println("/session_init - id:", record.SessionID, " - pid:", record.SessionPID) }() } diff --git a/pkg/sesswatch/sesswatch.go b/pkg/sesswatch/sesswatch.go index ae32bc4..3d786c5 100644 --- a/pkg/sesswatch/sesswatch.go +++ b/pkg/sesswatch/sesswatch.go @@ -34,7 +34,7 @@ func (s *sesswatch) waiter(sessionsToWatch chan records.Record, sessionsToWatchR s.mutex.Lock() defer s.mutex.Unlock() if s.watchedSessions[id] == false { - log.Println("sesswatch: start watching NEW session ~ pid:", id, "~", pid) + log.Println("sesswatch: start watching NEW session - id:", id, "; pid:", pid) s.watchedSessions[id] = true go s.watcher(id, pid) } @@ -45,7 +45,7 @@ func (s *sesswatch) waiter(sessionsToWatch chan records.Record, sessionsToWatchR s.mutex.Lock() defer s.mutex.Unlock() if s.watchedSessions[id] == false { - log.Println("sesswatch WARN: start watching NEW session (based on /record) ~ pid:", id, "~", pid) + log.Println("sesswatch WARN: start watching NEW session (based on /record) - id:", id, "; pid:", pid) s.watchedSessions[id] = true go s.watcher(id, pid) } @@ -59,9 +59,9 @@ func (s *sesswatch) watcher(sessionID string, sessionPID int) { time.Sleep(time.Duration(s.sleepSeconds) * time.Second) proc, err := ps.FindProcess(sessionPID) if err != nil { - log.Println("sesswatch ERROR: error while finding process:", sessionPID) + log.Println("sesswatch ERROR: error while finding process - pid:", sessionPID) } else if proc == nil { - log.Println("sesswatch: Dropping session ~ pid:", sessionID, "~", sessionPID) + log.Println("sesswatch: Dropping session - id:", sessionID, "; pid:", sessionPID) func() { s.mutex.Lock() defer s.mutex.Unlock() From ae9c182f932d6d573e94d801cd7cfb0d2f9e6a87 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 20 Apr 2022 02:25:44 +0200 Subject: [PATCH 11/35] fix error propagation, small cleanup --- pkg/histfile/histfile.go | 2 +- pkg/records/records.go | 8 ++++---- pkg/searchapp/test.go | 4 +--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/histfile/histfile.go b/pkg/histfile/histfile.go index 73e5309..17e7ffb 100644 --- a/pkg/histfile/histfile.go +++ b/pkg/histfile/histfile.go @@ -88,7 +88,7 @@ func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHi maxInitHistSize = math.MaxInt32 } log.Println("histfile: Loading resh history from file ...") - history := records.LoadFromFile(h.historyPath, math.MaxInt32) + history := records.LoadFromFile(h.historyPath) log.Println("histfile: resh history loaded from file - count:", len(history)) go h.loadCliRecords(history) // NOTE: keeping this weird interface for now because we might use it in the future diff --git a/pkg/records/records.go b/pkg/records/records.go index 6271ba5..411afd6 100644 --- a/pkg/records/records.go +++ b/pkg/records/records.go @@ -511,8 +511,8 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { } // LoadFromFile loads records from 'fname' file -func LoadFromFile(fname string, limit int) []Record { - const allowedErrors = 1 +func LoadFromFile(fname string) []Record { + const allowedErrors = 2 var encounteredErrors int // NOTE: limit does nothing atm var recs []Record @@ -528,7 +528,8 @@ func LoadFromFile(fname string, limit int) []Record { var i int var firstErrLine int for { - line, err := reader.ReadString('\n') + var line string + line, err = reader.ReadString('\n') if err != nil { break } @@ -553,7 +554,6 @@ func LoadFromFile(fname string, limit int) []Record { } recs = append(recs, record) } - // log.Println("records: done loading file:", err) if err != io.EOF { log.Println("records: error while loading file:", err) } diff --git a/pkg/searchapp/test.go b/pkg/searchapp/test.go index e33e2f7..30b850f 100644 --- a/pkg/searchapp/test.go +++ b/pkg/searchapp/test.go @@ -1,8 +1,6 @@ package searchapp import ( - "math" - "github.com/curusarn/resh/pkg/histcli" "github.com/curusarn/resh/pkg/msg" "github.com/curusarn/resh/pkg/records" @@ -10,7 +8,7 @@ import ( // LoadHistoryFromFile ... func LoadHistoryFromFile(historyPath string, numLines int) msg.CliResponse { - recs := records.LoadFromFile(historyPath, math.MaxInt32) + recs := records.LoadFromFile(historyPath) if numLines != 0 && numLines < len(recs) { recs = recs[:numLines] } From c1557bc900463c0ed3c6dcaaeb9d8f41e3983832 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 20 Apr 2022 02:26:29 +0200 Subject: [PATCH 12/35] go mod tidy --- go.mod | 3 --- go.sum | 7 ------- 2 files changed, 10 deletions(-) diff --git a/go.mod b/go.mod index 0ca87cd..ce2450a 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,10 @@ require ( github.com/awesome-gocui/gocui v1.0.0 github.com/coreos/go-semver v0.3.0 github.com/gdamore/tcell/v2 v2.4.0 // indirect - github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-shellwords v1.0.12 - github.com/mb-14/gomarkov v0.0.0-20210216094942-a5b484cc0243 github.com/mitchellh/go-ps v1.0.0 - github.com/schollz/progressbar v1.0.0 github.com/spf13/cobra v1.2.1 github.com/whilp/git-urls v1.0.0 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 diff --git a/go.sum b/go.sum index 60670d5..b44a8eb 100644 --- a/go.sum +++ b/go.sum @@ -170,8 +170,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 h1:1dSBUfGlorLAua2CRx0zFN7kQsTpE2DQSmr7rrTNgY8= -github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629/go.mod h1:mb5nS4uRANwOJSZj8rlCWAfAcGi72GGMIXx+xGOjA7M= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -195,8 +193,6 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mb-14/gomarkov v0.0.0-20210216094942-a5b484cc0243 h1:F0IAcxxFNzC8/HOxI5Q2hpsWAoGdy+lGMjoVyrcMeSw= -github.com/mb-14/gomarkov v0.0.0-20210216094942-a5b484cc0243/go.mod h1:5F3Y03oxWIyMq3Wa4AxU544RYnXNZwHBfqpDpdLibBY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -211,7 +207,6 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -226,8 +221,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= -github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= From 5ddb27b2e8ed7ac165eabbea1cc4b09f458e0176 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Thu, 21 Apr 2022 01:51:29 +0200 Subject: [PATCH 13/35] reshctl removal --- cmd/control/cmd/enable.go | 80 ---------------------------------- cmd/control/cmd/root.go | 10 +---- cmd/control/cmd/sanitize.go | 54 ----------------------- cmd/control/cmd/status.go | 72 ------------------------------- cmd/control/cmd/version.go | 86 +++++++++++++++++++++++++++++++++++++ pkg/records/records.go | 7 +-- scripts/hooks.sh | 4 -- scripts/install.sh | 4 -- scripts/reshctl.sh | 77 +-------------------------------- scripts/widgets.sh | 5 --- 10 files changed, 92 insertions(+), 307 deletions(-) delete mode 100644 cmd/control/cmd/enable.go delete mode 100644 cmd/control/cmd/sanitize.go delete mode 100644 cmd/control/cmd/status.go create mode 100644 cmd/control/cmd/version.go diff --git a/cmd/control/cmd/enable.go b/cmd/control/cmd/enable.go deleted file mode 100644 index 360be5b..0000000 --- a/cmd/control/cmd/enable.go +++ /dev/null @@ -1,80 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "os/user" - "path/filepath" - - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/cmd/control/status" - "github.com/curusarn/resh/pkg/cfg" - "github.com/spf13/cobra" -) - -// Enable commands - -var enableCmd = &cobra.Command{ - Use: "enable", - Short: "enable RESH features (bindings)", -} - -var enableControlRBindingCmd = &cobra.Command{ - Use: "ctrl_r_binding", - Short: "enable RESH-CLI binding for Ctrl+R", - Run: func(cmd *cobra.Command, args []string) { - exitCode = enableDisableControlRBindingGlobally(true) - if exitCode == status.Success { - exitCode = status.EnableControlRBinding - } - }, -} - -// Disable commands - -var disableCmd = &cobra.Command{ - Use: "disable", - Short: "disable RESH features (bindings)", -} - -var disableControlRBindingCmd = &cobra.Command{ - Use: "ctrl_r_binding", - Short: "disable RESH-CLI binding for Ctrl+R", - Run: func(cmd *cobra.Command, args []string) { - exitCode = enableDisableControlRBindingGlobally(false) - if exitCode == status.Success { - exitCode = status.DisableControlRBinding - } - }, -} - -func enableDisableControlRBindingGlobally(value bool) status.Code { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, ".config/resh.toml") - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - fmt.Println("Error reading config", err) - return status.Fail - } - if config.BindControlR != value { - config.BindControlR = value - - f, err := os.Create(configPath) - if err != nil { - fmt.Println("Error: Failed to create/open file:", configPath, "; error:", err) - return status.Fail - } - defer f.Close() - if err := toml.NewEncoder(f).Encode(config); err != nil { - fmt.Println("Error: Failed to encode and write the config values to hdd. error:", err) - return status.Fail - } - } - if value { - fmt.Println("RESH SEARCH app Ctrl+R binding: ENABLED") - } else { - fmt.Println("RESH SEARCH app Ctrl+R binding: DISABLED") - } - return status.Success -} diff --git a/cmd/control/cmd/root.go b/cmd/control/cmd/root.go index 35c770d..980d309 100644 --- a/cmd/control/cmd/root.go +++ b/cmd/control/cmd/root.go @@ -41,12 +41,6 @@ func Execute(ver, com string) status.Code { // log.SetFlags(log.LstdFlags | log.Lmicroseconds) } - rootCmd.AddCommand(enableCmd) - enableCmd.AddCommand(enableControlRBindingCmd) - - rootCmd.AddCommand(disableCmd) - disableCmd.AddCommand(disableControlRBindingCmd) - rootCmd.AddCommand(completionCmd) completionCmd.AddCommand(completionBashCmd) completionCmd.AddCommand(completionZshCmd) @@ -56,13 +50,11 @@ func Execute(ver, com string) status.Code { debugCmd.AddCommand(debugInspectCmd) debugCmd.AddCommand(debugOutputCmd) - rootCmd.AddCommand(statusCmd) + rootCmd.AddCommand(versionCmd) updateCmd.Flags().BoolVar(&betaFlag, "beta", false, "Update to latest version even if it's beta.") rootCmd.AddCommand(updateCmd) - rootCmd.AddCommand(sanitizeCmd) - if err := rootCmd.Execute(); err != nil { fmt.Println(err) return status.Fail diff --git a/cmd/control/cmd/sanitize.go b/cmd/control/cmd/sanitize.go deleted file mode 100644 index 08f29da..0000000 --- a/cmd/control/cmd/sanitize.go +++ /dev/null @@ -1,54 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "os/exec" - "os/user" - - "github.com/curusarn/resh/cmd/control/status" - "github.com/spf13/cobra" -) - -var sanitizeCmd = &cobra.Command{ - Use: "sanitize", - Short: "produce a sanitized version of your RESH history", - Run: func(cmd *cobra.Command, args []string) { - exitCode = status.Success - usr, _ := user.Current() - dir := usr.HomeDir - - fmt.Println() - fmt.Println(" HOW IT WORKS") - fmt.Println(" In sanitized history, all sensitive information is replaced with its SHA256 hashes.") - fmt.Println() - fmt.Println("Creating sanitized history files ...") - fmt.Println(" * ~/resh_history_sanitized.json (full lengh hashes)") - execCmd := exec.Command("resh-sanitize", "-trim-hashes", "0", "--output", dir+"/resh_history_sanitized.json") - execCmd.Stdout = os.Stdout - execCmd.Stderr = os.Stderr - err := execCmd.Run() - if err != nil { - exitCode = status.Fail - } - - fmt.Println(" * ~/resh_history_sanitized_trim12.json (12 char hashes)") - execCmd = exec.Command("resh-sanitize", "-trim-hashes", "12", "--output", dir+"/resh_history_sanitized_trim12.json") - execCmd.Stdout = os.Stdout - execCmd.Stderr = os.Stderr - err = execCmd.Run() - if err != nil { - exitCode = status.Fail - } - fmt.Println() - fmt.Println("Please direct all questions and/or issues to: https://github.com/curusarn/resh/issues") - fmt.Println() - fmt.Println("Please look at the resulting sanitized history using commands below.") - fmt.Println(" * Pretty print JSON") - fmt.Println(" cat ~/resh_history_sanitized_trim12.json | jq") - fmt.Println() - fmt.Println(" * Only show commands, don't show metadata") - fmt.Println(" cat ~/resh_history_sanitized_trim12.json | jq '.[\"cmdLine\"]'") - fmt.Println() - }, -} diff --git a/cmd/control/cmd/status.go b/cmd/control/cmd/status.go deleted file mode 100644 index f12b7fa..0000000 --- a/cmd/control/cmd/status.go +++ /dev/null @@ -1,72 +0,0 @@ -package cmd - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "strconv" - - "github.com/curusarn/resh/cmd/control/status" - "github.com/curusarn/resh/pkg/msg" - "github.com/spf13/cobra" -) - -var statusCmd = &cobra.Command{ - Use: "status", - Short: "show RESH status", - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("resh " + version) - fmt.Println() - fmt.Println("Resh versions ...") - fmt.Println(" * installed: " + version + " (" + commit + ")") - versionEnv, found := os.LookupEnv("__RESH_VERSION") - if found == false { - versionEnv = "UNKNOWN!" - } - commitEnv, found := os.LookupEnv("__RESH_REVISION") - if found == false { - commitEnv = "unknown" - } - fmt.Println(" * this shell session: " + versionEnv + " (" + commitEnv + ")") - - resp, err := getDaemonStatus(config.Port) - if err != nil { - fmt.Println(" * RESH-DAEMON IS NOT RUNNING") - fmt.Println(" * Please REPORT this here: https://github.com/curusarn/resh/issues") - fmt.Println(" * Please RESTART this terminal window") - exitCode = status.Fail - return - } - fmt.Println(" * daemon: " + resp.Version + " (" + resp.Commit + ")") - - if version != resp.Version || version != versionEnv { - fmt.Println(" * THERE IS A MISMATCH BETWEEN VERSIONS!") - fmt.Println(" * Please REPORT this here: https://github.com/curusarn/resh/issues") - fmt.Println(" * Please RESTART this terminal window") - } - - exitCode = status.ReshStatus - }, -} - -func getDaemonStatus(port int) (msg.StatusResponse, error) { - mess := msg.StatusResponse{} - url := "http://localhost:" + strconv.Itoa(port) + "/status" - resp, err := http.Get(url) - if err != nil { - return mess, err - } - defer resp.Body.Close() - jsn, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Fatal("Error while reading 'daemon /status' response:", err) - } - err = json.Unmarshal(jsn, &mess) - if err != nil { - log.Fatal("Error while decoding 'daemon /status' response:", err) - } - return mess, nil -} diff --git a/cmd/control/cmd/version.go b/cmd/control/cmd/version.go new file mode 100644 index 0000000..c9d1edf --- /dev/null +++ b/cmd/control/cmd/version.go @@ -0,0 +1,86 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + + "github.com/curusarn/resh/cmd/control/status" + "github.com/curusarn/resh/pkg/msg" + "github.com/spf13/cobra" +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "show RESH version", + Run: func(cmd *cobra.Command, args []string) { + printVersion("Installed", version, commit) + + versionEnv := getEnvVarWithDefault("__RESH_VERSION", "") + commitEnv := getEnvVarWithDefault("__RESH_REVISION", "") + printVersion("This terminal session", versionEnv, commitEnv) + + resp, err := getDaemonStatus(config.Port) + if err != nil { + fmt.Fprintf(os.Stderr, "\nERROR: Resh-daemon didn't respond - it's probably not running.\n\n") + fmt.Fprintf(os.Stderr, "-> Try restarting this terminal window to bring resh-daemon back up.\n") + fmt.Fprintf(os.Stderr, "-> If the problem persists you can check resh-daemon logs: ~/.resh/daemon.log\n") + exitCode = status.Fail + return + } + printVersion("Daemon", resp.Version, resp.Commit) + + if version != versionEnv { + fmt.Fprintf(os.Stderr, "\nWARN: This terminal session was started with different resh version than is installed now - it looks like you updated resh and didn't restart this terminal.\n\n") + fmt.Fprintf(os.Stderr, "-> Restart this terminal window to fix that.\n") + return + } + if version != resp.Version { + fmt.Fprintf(os.Stderr, "\nWARN: Resh-daemon is running in different version than is installed now - it looks like something went wrong during resh update.\n\n") + fmt.Fprintf(os.Stderr, "-> Kill resh-daemon and then launch a new terminal window to fix that.\n") + fmt.Fprintf(os.Stderr, " $ pkill resh-daemon\n") + return + } + + exitCode = status.ReshStatus + }, +} + +func printErrExtras() { + +} + +func printVersion(title, version, commit string) { + fmt.Printf("%s: %s (commit: %s)\n", title, version, commit) +} + +func getEnvVarWithDefault(varName, defaultValue string) string { + val, found := os.LookupEnv(varName) + if !found { + return defaultValue + } + return val +} + +func getDaemonStatus(port int) (msg.StatusResponse, error) { + mess := msg.StatusResponse{} + url := "http://localhost:" + strconv.Itoa(port) + "/status" + resp, err := http.Get(url) + if err != nil { + return mess, err + } + defer resp.Body.Close() + jsn, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatal("Error while reading 'daemon /status' response:", err) + } + err = json.Unmarshal(jsn, &mess) + if err != nil { + log.Fatal("Error while decoding 'daemon /status' response:", err) + } + return mess, nil +} diff --git a/pkg/records/records.go b/pkg/records/records.go index 411afd6..ef0f561 100644 --- a/pkg/records/records.go +++ b/pkg/records/records.go @@ -560,9 +560,10 @@ func LoadFromFile(fname string) []Record { // log.Println("records: Loaded lines - count:", i) if encounteredErrors > 0 { // fix errors in the history file - log.Printf("There were %d decoding errors, the first error happend on line %d/%d", encounteredErrors, firstErrLine, i) - log.Println("Backing up current history file ...") - err := copyFile(fname, fname+".bak") + log.Printf("There were %d decoding errors, the first error happend on line %d/%d\n", encounteredErrors, firstErrLine, i) + fnameBak := fname + ".bak" + log.Printf("Backing up current history file to %s\n", fnameBak) + err := copyFile(fname, fnameBak) if err != nil { log.Fatalln("Failed to backup history file with decode errors") } diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 25e7e02..146ddf9 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -18,9 +18,7 @@ __resh_collect() { local __RESH_COLS="$COLUMNS" local __RESH_LANG="$LANG" local __RESH_LC_ALL="$LC_ALL" - # other LC ? local __RESH_LINES="$LINES" - # __RESH_PATH="$PATH" local __RESH_PWD="$PWD" # non-posix @@ -29,8 +27,6 @@ __resh_collect() { local __RESH_GIT_CDUP_EXIT_CODE=$? local __RESH_GIT_REMOTE; __RESH_GIT_REMOTE="$(git remote get-url origin 2>/dev/null)" local __RESH_GIT_REMOTE_EXIT_CODE=$? - #__RESH_GIT_TOPLEVEL="$(git rev-parse --show-toplevel)" - #__RESH_GIT_TOPLEVEL_EXIT_CODE=$? if [ -n "${ZSH_VERSION-}" ]; then # assume Zsh diff --git a/scripts/install.sh b/scripts/install.sh index 080e176..5caedca 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -218,10 +218,6 @@ RESH SEARCH APPLICATION = Redesigned reverse search that actually works At first, the search application will use the standard shell history without context. All history recorded from now on will have context which will by the RESH SEARCH app. - Enable/disable Ctrl+R binding using reshctl command: - $ reshctl enable ctrl_r_binding - $ reshctl disable ctrl_r_binding - CHECK FOR UPDATES To check for (and install) updates use reshctl command: $ reshctl update diff --git a/scripts/reshctl.sh b/scripts/reshctl.sh index db02147..31d1a0f 100644 --- a/scripts/reshctl.sh +++ b/scripts/reshctl.sh @@ -77,79 +77,4 @@ resh() { echo "$buffer" >| "$fpath_last_run" echo "resh-cli failed - check '$fpath_last_run' and '~/.resh/cli.log'" fi -} - -reshctl() { - # export current shell because resh-control needs to know - export __RESH_ctl_shell=$__RESH_SHELL - # run resh-control aka the real reshctl - resh-control "$@" - - # modify current shell session based on exit status - local _status=$? - # echo $_status - # unexport current shell - unset __RESH_ctl_shell - case "$_status" in - 0|1) - # success | fail - return "$_status" - ;; - # enable - # 30) - # # enable all - # __resh_bind_all - # return 0 - # ;; - 32) - # enable control R - __resh_bind_control_R - return 0 - ;; - # disable - # 40) - # # disable all - # __resh_unbind_all - # return 0 - # ;; - 42) - # disable control R - __resh_unbind_control_R - return 0 - ;; - 50) - # reload rc files - . ~/.resh/shellrc - return 0 - ;; - 51) - # inspect session history - # reshctl debug inspect N - resh-inspect --sessionID "$__RESH_SESSION_ID" --count "${3-10}" - return 0 - ;; - 52) - # show status - echo - echo 'Control R binding ...' - if [ "$(resh-config --key BindControlR)" = true ]; then - echo ' * future sessions: ENABLED' - else - echo ' * future sessions: DISABLED' - fi - if [ "${__RESH_control_R_bind_enabled-0}" != 0 ]; then - echo ' * this session: ENABLED' - else - echo ' * this session: DISABLED' - fi - return 0 - ;; - *) - echo "reshctl() FATAL ERROR: unknown status ($_status)" >&2 - echo "Possibly caused by version mismatch between installed resh and resh in this session." >&2 - echo "Please REPORT this issue here: https://github.com/curusarn/resh/issues" >&2 - echo "Please RESTART your terminal window." >&2 - return "$_status" - ;; - esac -} +} \ No newline at end of file diff --git a/scripts/widgets.sh b/scripts/widgets.sh index defc605..26f5b07 100644 --- a/scripts/widgets.sh +++ b/scripts/widgets.sh @@ -9,10 +9,7 @@ __resh_widget_control_R() { # shellcheck disable=2034 __bp_preexec_interactive_mode="on" - # local __RESH_PREFIX=${BUFFER:0:CURSOR} - # __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS;control_R:$__RESH_PREFIX" local PREVBUFFER=$BUFFER - __RESH_HIST_RECALL_ACTIONS="$__RESH_HIST_RECALL_ACTIONS|||control_R:$BUFFER" local status_code local git_remote; git_remote="$(git remote get-url origin 2>/dev/null)" @@ -42,8 +39,6 @@ __resh_widget_control_R() { BUFFER="$PREVBUFFER" fi CURSOR=${#BUFFER} - # recorded to history - __RESH_HIST_PREV_LINE=${BUFFER} } __resh_widget_control_R_compat() { From e06fe26375b62ca5f963ac31ec65518164a3a0a5 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 23 Apr 2022 23:30:07 +0200 Subject: [PATCH 14/35] copy --- Makefile | 4 ++-- cmd/control/cmd/version.go | 18 ++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 251527b..f90918c 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ SHELL=/bin/bash LATEST_TAG=$(shell git describe --tags) -REVISION=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_revision") +COMMIT=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_commit") VERSION="${LATEST_TAG}-DEV" -GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${REVISION}" +GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT}" build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect bin/resh-daemon\ diff --git a/cmd/control/cmd/version.go b/cmd/control/cmd/version.go index c9d1edf..57c776f 100644 --- a/cmd/control/cmd/version.go +++ b/cmd/control/cmd/version.go @@ -29,20 +29,22 @@ var versionCmd = &cobra.Command{ fmt.Fprintf(os.Stderr, "\nERROR: Resh-daemon didn't respond - it's probably not running.\n\n") fmt.Fprintf(os.Stderr, "-> Try restarting this terminal window to bring resh-daemon back up.\n") fmt.Fprintf(os.Stderr, "-> If the problem persists you can check resh-daemon logs: ~/.resh/daemon.log\n") + fmt.Fprintf(os.Stderr, "-> You can file an issue at: https://github.com/curusarn/resh/issues\n") exitCode = status.Fail return } - printVersion("Daemon", resp.Version, resp.Commit) + printVersion("Currently running daemon", resp.Version, resp.Commit) - if version != versionEnv { - fmt.Fprintf(os.Stderr, "\nWARN: This terminal session was started with different resh version than is installed now - it looks like you updated resh and didn't restart this terminal.\n\n") - fmt.Fprintf(os.Stderr, "-> Restart this terminal window to fix that.\n") - return - } if version != resp.Version { fmt.Fprintf(os.Stderr, "\nWARN: Resh-daemon is running in different version than is installed now - it looks like something went wrong during resh update.\n\n") fmt.Fprintf(os.Stderr, "-> Kill resh-daemon and then launch a new terminal window to fix that.\n") fmt.Fprintf(os.Stderr, " $ pkill resh-daemon\n") + fmt.Fprintf(os.Stderr, "-> You can file an issue at: https://github.com/curusarn/resh/issues\n") + return + } + if version != versionEnv { + fmt.Fprintf(os.Stderr, "\nWARN: This terminal session was started with different resh version than is installed now - it looks like you updated resh and didn't restart this terminal.\n\n") + fmt.Fprintf(os.Stderr, "-> Restart this terminal window to fix that.\n") return } @@ -50,10 +52,6 @@ var versionCmd = &cobra.Command{ }, } -func printErrExtras() { - -} - func printVersion(title, version, commit string) { fmt.Printf("%s: %s (commit: %s)\n", title, version, commit) } From f5349fc45bbbdbd8730f40ce936ac9b164bedf56 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 3 May 2022 02:52:54 +0200 Subject: [PATCH 15/35] structured logging, refactoring, improvements --- Makefile | 5 +- cmd/cli/main.go | 242 ++++---- cmd/collect/main.go | 54 +- cmd/config/main.go | 26 +- cmd/control/cmd/completion.go | 3 - cmd/control/cmd/debug.go | 66 --- cmd/control/cmd/root.go | 46 +- cmd/control/cmd/update.go | 16 +- cmd/control/cmd/version.go | 11 +- cmd/control/main.go | 4 +- cmd/control/status/status.go | 25 - cmd/daemon/dump.go | 35 +- cmd/daemon/kill.go | 28 - cmd/daemon/main.go | 160 ++++-- cmd/daemon/record.go | 35 +- cmd/daemon/run-server.go | 48 +- cmd/daemon/session-init.go | 24 +- cmd/daemon/status.go | 36 +- cmd/postcollect/main.go | 50 +- cmd/sanitize/main.go | 523 ------------------ cmd/session-init/main.go | 49 +- conf/config.toml | 2 +- go.mod | 20 +- go.sum | 18 +- internal/cfg/cfg.go | 117 ++++ {pkg => internal}/collect/collect.go | 31 +- {pkg => internal}/histcli/histcli.go | 2 +- {pkg => internal}/histfile/histfile.go | 148 ++--- {pkg => internal}/histlist/histlist.go | 28 +- {pkg => internal}/httpclient/httpclient.go | 0 internal/logger/logger.go | 28 + {pkg => internal}/msg/msg.go | 2 +- internal/output/output.go | 69 +++ {pkg => internal}/records/records.go | 89 +-- {pkg => internal}/records/records_test.go | 5 +- .../records/testdata/resh_history.json | 0 {pkg => internal}/searchapp/highlight.go | 0 {pkg => internal}/searchapp/item.go | 6 +- {pkg => internal}/searchapp/item_test.go | 0 {pkg => internal}/searchapp/query.go | 17 - internal/searchapp/test.go | 22 + {pkg => internal}/searchapp/time.go | 0 {pkg => internal}/sess/sess.go | 0 {pkg => internal}/sesswatch/sesswatch.go | 44 +- internal/signalhandler/signalhander.go | 74 +++ pkg/cfg/cfg.go | 12 - pkg/searchapp/test.go | 21 - pkg/signalhandler/signalhander.go | 65 --- scripts/util.sh | 2 + 49 files changed, 987 insertions(+), 1321 deletions(-) delete mode 100644 cmd/control/cmd/debug.go delete mode 100644 cmd/control/status/status.go delete mode 100644 cmd/daemon/kill.go delete mode 100644 cmd/sanitize/main.go create mode 100644 internal/cfg/cfg.go rename {pkg => internal}/collect/collect.go (59%) rename {pkg => internal}/histcli/histcli.go (92%) rename {pkg => internal}/histfile/histfile.go (57%) rename {pkg => internal}/histlist/histlist.go (62%) rename {pkg => internal}/httpclient/httpclient.go (100%) create mode 100644 internal/logger/logger.go rename {pkg => internal}/msg/msg.go (92%) create mode 100644 internal/output/output.go rename {pkg => internal}/records/records.go (88%) rename {pkg => internal}/records/records_test.go (96%) rename {pkg => internal}/records/testdata/resh_history.json (100%) rename {pkg => internal}/searchapp/highlight.go (100%) rename {pkg => internal}/searchapp/item.go (99%) rename {pkg => internal}/searchapp/item_test.go (100%) rename {pkg => internal}/searchapp/query.go (80%) create mode 100644 internal/searchapp/test.go rename {pkg => internal}/searchapp/time.go (100%) rename {pkg => internal}/sess/sess.go (100%) rename {pkg => internal}/sesswatch/sesswatch.go (55%) create mode 100644 internal/signalhandler/signalhander.go delete mode 100644 pkg/cfg/cfg.go delete mode 100644 pkg/searchapp/test.go delete mode 100644 pkg/signalhandler/signalhander.go diff --git a/Makefile b/Makefile index f90918c..b269c6d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL=/bin/bash LATEST_TAG=$(shell git describe --tags) COMMIT=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_commit") VERSION="${LATEST_TAG}-DEV" -GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT}" +GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.development=true" build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect bin/resh-daemon\ @@ -27,7 +27,8 @@ uninstall: # Uninstalling ... -rm -rf ~/.resh/ -bin/resh-%: cmd/%/*.go pkg/*/*.go cmd/control/cmd/*.go cmd/control/status/status.go +go_files = $(shell find -name '*.go') +bin/resh-%: $(go_files) grep $@ .goreleaser.yml -q # all build targets need to be included in .goreleaser.yml go build ${GOFLAGS} -o $@ cmd/$*/*.go diff --git a/cmd/cli/main.go b/cmd/cli/main.go index a58ff10..9ea53ae 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io/ioutil" - "log" "net/http" "os" "sort" @@ -15,59 +14,41 @@ import ( "sync" "time" - "github.com/BurntSushi/toml" "github.com/awesome-gocui/gocui" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/msg" - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/searchapp" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/msg" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/searchapp" + "go.uber.org/zap" - "os/user" - "path/filepath" "strconv" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string +var developement bool // special constant recognized by RESH wrappers const exitCodeExecute = 111 -var debug bool - func main() { - output, exitCode := runReshCli() + config, errCfg := cfg.New() + logger, _ := logger.New("search-app", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) + } + out := output.New(logger, "resh-search-app ERROR") + + output, exitCode := runReshCli(out, config) fmt.Print(output) os.Exit(exitCode) } -func runReshCli() (string, int) { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - logPath := filepath.Join(dir, ".resh/cli.log") - - f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - log.Fatal("Error opening file:", err) - } - defer f.Close() - - log.SetOutput(f) - - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) - } - if config.Debug { - debug = true - log.SetFlags(log.LstdFlags | log.Lmicroseconds) - log.Println("DEBUG is ON") - } - +func runReshCli(out *output.Output, config cfg.Config) (string, int) { sessionID := flag.String("sessionID", "", "resh generated session id") host := flag.String("host", "", "host") pwd := flag.String("pwd", "", "present working directory") @@ -77,22 +58,23 @@ func runReshCli() (string, int) { testHistoryLines := flag.Int("test-lines", 0, "the number of lines to load from a file passed with --test-history (for testing purposes only!)") flag.Parse() + errMsg := "Failed to get necessary command-line arguments" if *sessionID == "" { - log.Println("Error: you need to specify sessionId") + out.Fatal(errMsg, errors.New("missing option --sessionId")) } if *host == "" { - log.Println("Error: you need to specify HOST") + out.Fatal(errMsg, errors.New("missing option --host")) } if *pwd == "" { - log.Println("Error: you need to specify PWD") + out.Fatal(errMsg, errors.New("missing option --pwd")) } if *gitOriginRemote == "DEFAULT" { - log.Println("Error: you need to specify gitOriginRemote") + out.Fatal(errMsg, errors.New("missing option --gitOriginRemote")) } g, err := gocui.NewGui(gocui.OutputNormal, false) if err != nil { - log.Panicln(err) + out.Fatal("Failed to launch TUI", err) } defer g.Close() @@ -107,9 +89,9 @@ func runReshCli() (string, int) { SessionID: *sessionID, PWD: *pwd, } - resp = SendCliMsg(mess, strconv.Itoa(config.Port)) + resp = SendCliMsg(out, mess, strconv.Itoa(config.Port)) } else { - resp = searchapp.LoadHistoryFromFile(*testHistory, *testHistoryLines) + resp = searchapp.LoadHistoryFromFile(out.Logger.Sugar(), *testHistory, *testHistoryLines) } st := state{ @@ -128,47 +110,48 @@ func runReshCli() (string, int) { } g.SetManager(layout) + errMsg = "Failed to set keybindings" if err := g.SetKeybinding("", gocui.KeyTab, gocui.ModNone, layout.Next); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyArrowDown, gocui.ModNone, layout.Next); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlN, gocui.ModNone, layout.Next); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyArrowUp, gocui.ModNone, layout.Prev); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlP, gocui.ModNone, layout.Prev); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyArrowRight, gocui.ModNone, layout.SelectPaste); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyEnter, gocui.ModNone, layout.SelectExecute); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlG, gocui.ModNone, layout.AbortPaste); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlD, gocui.ModNone, quit); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } if err := g.SetKeybinding("", gocui.KeyCtrlR, gocui.ModNone, layout.SwitchModes); err != nil { - log.Panicln(err) + out.Fatal(errMsg, err) } layout.UpdateData(*query) layout.UpdateRawData(*query) err = g.MainLoop() if err != nil && !errors.Is(err, gocui.ErrQuit) { - log.Panicln(err) + out.Fatal("Main application loop finished with error", err) } return layout.s.output, layout.s.exitCode } @@ -190,11 +173,13 @@ type state struct { } type manager struct { + out *output.Output + config cfg.Config + sessionID string host string pwd string gitOriginRemote string - config cfg.Config s *state } @@ -254,11 +239,11 @@ type dedupRecord struct { } func (m manager) UpdateData(input string) { - if debug { - log.Println("EDIT start") - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - } + sugar := m.out.Logger.Sugar() + sugar.Debugw("Starting data update ...", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) query := searchapp.NewQueryFromString(input, m.host, m.pwd, m.gitOriginRemote, m.config.Debug) var data []searchapp.Item itemSet := make(map[string]int) @@ -268,7 +253,7 @@ func (m manager) UpdateData(input string) { itm, err := searchapp.NewItemFromRecordForQuery(rec, query, m.config.Debug) if err != nil { // records didn't match the query - // log.Println(" * continue (no match)", rec.Pwd) + // sugar.Println(" * continue (no match)", rec.Pwd) continue } if idx, ok := itemSet[itm.Key]; ok { @@ -285,9 +270,9 @@ func (m manager) UpdateData(input string) { itemSet[itm.Key] = len(data) data = append(data, itm) } - if debug { - log.Println("len(tmpdata) =", len(data)) - } + sugar.Debugw("Got new items from records for query, sorting items ...", + "itemCount", len(data), + ) sort.SliceStable(data, func(p, q int) bool { return data[p].Score > data[q].Score }) @@ -299,19 +284,18 @@ func (m manager) UpdateData(input string) { m.s.data = append(m.s.data, itm) } m.s.highlightedItem = 0 - if debug { - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - log.Println("EDIT end") - } + sugar.Debugw("Done with data update", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) } func (m manager) UpdateRawData(input string) { - if m.config.Debug { - log.Println("EDIT start") - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - } + sugar := m.out.Logger.Sugar() + sugar.Debugw("Starting RAW data update ...", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) query := searchapp.GetRawTermsFromString(input, m.config.Debug) var data []searchapp.RawItem itemSet := make(map[string]bool) @@ -321,20 +305,20 @@ func (m manager) UpdateRawData(input string) { itm, err := searchapp.NewRawItemFromRecordForQuery(rec, query, m.config.Debug) if err != nil { // records didn't match the query - // log.Println(" * continue (no match)", rec.Pwd) + // sugar.Println(" * continue (no match)", rec.Pwd) continue } if itemSet[itm.Key] { - // log.Println(" * continue (already present)", itm.key(), itm.pwd) + // sugar.Println(" * continue (already present)", itm.key(), itm.pwd) continue } itemSet[itm.Key] = true data = append(data, itm) - // log.Println("DATA =", itm.display) - } - if debug { - log.Println("len(tmpdata) =", len(data)) + // sugar.Println("DATA =", itm.display) } + sugar.Debugw("Got new RAW items from records for query, sorting items ...", + "itemCount", len(data), + ) sort.SliceStable(data, func(p, q int) bool { return data[p].Score > data[q].Score }) @@ -346,11 +330,10 @@ func (m manager) UpdateRawData(input string) { m.s.rawData = append(m.s.rawData, itm) } m.s.highlightedItem = 0 - if debug { - log.Println("len(fullRecords) =", len(m.s.cliRecords)) - log.Println("len(data) =", len(m.s.data)) - log.Println("EDIT end") - } + sugar.Debugw("Done with RAW data update", + "recordCount", len(m.s.cliRecords), + "itemCount", len(m.s.data), + ) } func (m manager) Edit(v *gocui.View, key gocui.Key, ch rune, mod gocui.Modifier) { gocui.DefaultEditor.Edit(v, key, ch, mod) @@ -398,7 +381,7 @@ func (m manager) Layout(g *gocui.Gui) error { v, err := g.SetView("input", 0, 0, maxX-1, 2, b) if err != nil && !errors.Is(err, gocui.ErrUnknownView) { - log.Panicln(err.Error()) + m.out.Fatal("Failed to set view 'input'", err) } v.Editable = true @@ -421,7 +404,7 @@ func (m manager) Layout(g *gocui.Gui) error { v, err = g.SetView("body", 0, 2, maxX-1, maxY, b) if err != nil && !errors.Is(err, gocui.ErrUnknownView) { - log.Panicln(err.Error()) + m.out.Fatal("Failed to set view 'body'", err) } v.Frame = false v.Autoscroll = false @@ -441,6 +424,7 @@ func quit(g *gocui.Gui, v *gocui.View) error { const smallTerminalTresholdWidth = 110 func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { + sugar := m.out.Logger.Sugar() maxX, maxY := g.Size() compactRenderingMode := false @@ -459,7 +443,7 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { if i == maxY { break } - ic := itm.DrawItemColumns(compactRenderingMode, debug) + ic := itm.DrawItemColumns(compactRenderingMode, m.config.Debug) data = append(data, ic) if i > maxPossibleMainViewHeight { // do not stretch columns because of results that will end up outside of the page @@ -505,7 +489,7 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { // header // header := getHeader() // error is expected for header - dispStr, _, _ := header.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, true, true, debug) + dispStr, _, _ := header.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, true, true, m.config.Debug) dispStr = searchapp.DoHighlightHeader(dispStr, maxX*2) v.WriteString(dispStr + "\n") @@ -513,33 +497,24 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { for index < len(data) { itm := data[index] if index >= mainViewHeight { - if debug { - log.Printf("Finished drawing page. mainViewHeight: %v, predictedMax: %v\n", - mainViewHeight, maxPossibleMainViewHeight) - } + sugar.Debugw("Reached bottom of the page while producing lines", + "mainViewHeight", mainViewHeight, + "predictedMaxViewHeight", maxPossibleMainViewHeight, + ) // page is full break } - displayStr, _, err := itm.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, false, true, debug) + displayStr, _, err := itm.ProduceLine(longestDateLen, longestLocationLen, longestFlagsLen, false, true, m.config.Debug) if err != nil { - log.Printf("produceLine error: %v\n", err) + sugar.Error("Error while drawing item", zap.Error(err)) } if m.s.highlightedItem == index { // maxX * 2 because there are escape sequences that make it hard to tell the real string length displayStr = searchapp.DoHighlightString(displayStr, maxX*3) - if debug { - log.Println("### HightlightedItem string :", displayStr) - } - } else if debug { - log.Println(displayStr) } if strings.Contains(displayStr, "\n") { - log.Println("display string contained \\n") displayStr = strings.ReplaceAll(displayStr, "\n", "#") - if debug { - log.Println("display string contained \\n") - } } v.WriteString(displayStr + "\n") index++ @@ -553,59 +528,46 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { v.WriteString(line) } v.WriteString(helpLine) - if debug { - log.Println("len(data) =", len(m.s.data)) - log.Println("highlightedItem =", m.s.highlightedItem) - } + sugar.Debugw("Done drawing page", + "itemCount", len(m.s.data), + "highlightedItemIndex", m.s.highlightedItem, + ) return nil } func (m manager) rawMode(g *gocui.Gui, v *gocui.View) error { + sugar := m.out.Logger.Sugar() maxX, maxY := g.Size() topBoxSize := 3 m.s.displayedItemsCount = maxY - topBoxSize for i, itm := range m.s.rawData { if i == maxY { - if debug { - log.Println(maxY) - } break } displayStr := itm.CmdLineWithColor if m.s.highlightedItem == i { // use actual min requried length instead of 420 constant displayStr = searchapp.DoHighlightString(displayStr, maxX*2) - if debug { - log.Println("### HightlightedItem string :", displayStr) - } - } else if debug { - log.Println(displayStr) } if strings.Contains(displayStr, "\n") { - log.Println("display string contained \\n") displayStr = strings.ReplaceAll(displayStr, "\n", "#") - if debug { - log.Println("display string contained \\n") - } } v.WriteString(displayStr + "\n") - // if m.s.highlightedItem == i { - // v.SetHighlight(m.s.highlightedItem, true) - // } - } - if debug { - log.Println("len(data) =", len(m.s.data)) - log.Println("highlightedItem =", m.s.highlightedItem) } + sugar.Debugw("Done drawing page in RAW mode", + "itemCount", len(m.s.data), + "highlightedItemIndex", m.s.highlightedItem, + ) return nil } // SendCliMsg to daemon -func SendCliMsg(m msg.CliMsg, port string) msg.CliResponse { +func SendCliMsg(out *output.Output, m msg.CliMsg, port string) msg.CliResponse { + sugar := out.Logger.Sugar() recJSON, err := json.Marshal(m) if err != nil { - log.Fatalf("Failed to marshal message: %v\n", err) + out.Fatal("Failed to marshal message", err) } req, err := http.NewRequest( @@ -613,7 +575,7 @@ func SendCliMsg(m msg.CliMsg, port string) msg.CliResponse { "http://localhost:"+port+"/dump", bytes.NewBuffer(recJSON)) if err != nil { - log.Fatalf("Failed to build request: %v\n", err) + out.Fatal("Failed to build request", err) } req.Header.Set("Content-Type", "application/json") @@ -622,22 +584,22 @@ func SendCliMsg(m msg.CliMsg, port string) msg.CliResponse { } resp, err := client.Do(req) if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") + out.FatalDaemonNotRunning(err) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Fatalf("Read response error: %v\n", err) + out.Fatal("Failed read response", err) } - // log.Println(string(body)) + // sugar.Println(string(body)) response := msg.CliResponse{} err = json.Unmarshal(body, &response) if err != nil { - log.Fatalf("Unmarshal resp error: %v\n", err) - } - if debug { - log.Printf("Recieved %d records from daemon\n", len(response.CliRecords)) + out.Fatal("Failed decode response", err) } + sugar.Debug("Recieved records from daemon", + "recordCount", len(response.CliRecords), + ) return response } diff --git a/cmd/collect/main.go b/cmd/collect/main.go index 57e6135..a0a17b2 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -3,39 +3,43 @@ package main import ( "flag" "fmt" - "log" "os" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/collect" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" // "os/exec" - "os/user" + "path/filepath" "strconv" ) -// version tag from git set during build +// info passed during build var version string - -// Commit hash from git set during build var commit string +var developement bool func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") - - machineIDPath := "/etc/machine-id" + config, errCfg := cfg.New() + logger, _ := logger.New("collect", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) + } + out := output.New(logger, "resh-collect ERROR") - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) + homeDir, err := os.UserHomeDir() + if err != nil { + out.Fatal("Could not get user home dir", err) } + reshUUIDPath := filepath.Join(homeDir, "/.resh/resh-uuid") + machineIDPath := "/etc/machine-id" + // version showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") @@ -121,29 +125,29 @@ func main() { realtimeBefore, err := strconv.ParseFloat(*rtb, 64) if err != nil { - log.Fatal("Flag Parsing error (rtb):", err) + out.Fatal("Error while parsing flag --realtimeBefore", err) } realtimeSessionStart, err := strconv.ParseFloat(*rtsess, 64) if err != nil { - log.Fatal("Flag Parsing error (rt sess):", err) + out.Fatal("Error while parsing flag --realtimeSession", err) } realtimeSessSinceBoot, err := strconv.ParseFloat(*rtsessboot, 64) if err != nil { - log.Fatal("Flag Parsing error (rt sess boot):", err) + out.Fatal("Error while parsing flag --realtimeSessSinceBoot", err) } realtimeSinceSessionStart := realtimeBefore - realtimeSessionStart realtimeSinceBoot := realtimeSessSinceBoot + realtimeSinceSessionStart - timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(*timezoneBefore) + timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(logger, *timezoneBefore) realtimeBeforeLocal := realtimeBefore + timezoneBeforeOffset realPwd, err := filepath.EvalSymlinks(*pwd) if err != nil { - log.Println("err while handling pwd realpath:", err) + logger.Error("Error while handling pwd realpath", zap.Error(err)) realPwd = "" } - gitDir, gitRealDir := collect.GetGitDirs(*gitCdup, *gitCdupExitCode, *pwd) + gitDir, gitRealDir := collect.GetGitDirs(logger, *gitCdup, *gitCdupExitCode, *pwd) if *gitRemoteExitCode != 0 { *gitRemote = "" } @@ -218,5 +222,5 @@ func main() { ReshRevision: commit, }, } - collect.SendRecord(rec, strconv.Itoa(config.Port), "/record") + collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/record") } diff --git a/cmd/config/main.go b/cmd/config/main.go index e67ee7b..e908817 100644 --- a/cmd/config/main.go +++ b/cmd/config/main.go @@ -4,24 +4,24 @@ import ( "flag" "fmt" "os" - "os/user" - "path/filepath" "strings" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/logger" + "go.uber.org/zap" ) +// info passed during build +var version string +var commit string +var developement bool + func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, ".config/resh.toml") - - var config cfg.Config - _, err := toml.DecodeFile(configPath, &config) - if err != nil { - fmt.Println("Error reading config", err) - os.Exit(1) + config, errCfg := cfg.New() + logger, _ := logger.New("config", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) } configKey := flag.String("key", "", "Key of the requested config entry") diff --git a/cmd/control/cmd/completion.go b/cmd/control/cmd/completion.go index e80fa56..6c0e450 100644 --- a/cmd/control/cmd/completion.go +++ b/cmd/control/cmd/completion.go @@ -3,7 +3,6 @@ package cmd import ( "os" - "github.com/curusarn/resh/cmd/control/status" "github.com/spf13/cobra" ) @@ -30,7 +29,6 @@ var completionBashCmd = &cobra.Command{ `, Run: func(cmd *cobra.Command, args []string) { rootCmd.GenBashCompletion(os.Stdout) - exitCode = status.Success }, } @@ -43,6 +41,5 @@ var completionZshCmd = &cobra.Command{ `, Run: func(cmd *cobra.Command, args []string) { rootCmd.GenZshCompletion(os.Stdout) - exitCode = status.Success }, } diff --git a/cmd/control/cmd/debug.go b/cmd/control/cmd/debug.go deleted file mode 100644 index f1401c7..0000000 --- a/cmd/control/cmd/debug.go +++ /dev/null @@ -1,66 +0,0 @@ -package cmd - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/curusarn/resh/cmd/control/status" - "github.com/spf13/cobra" -) - -var debugCmd = &cobra.Command{ - Use: "debug", - Short: "debug utils for resh", - Long: "Reloads resh rc files. Shows logs and output from last runs of resh", -} - -var debugReloadCmd = &cobra.Command{ - Use: "reload", - Short: "reload resh rc files", - Long: "Reload resh rc files", - Run: func(cmd *cobra.Command, args []string) { - exitCode = status.ReloadRcFiles - }, -} - -var debugInspectCmd = &cobra.Command{ - Use: "inspect", - Short: "inspect session history", - Run: func(cmd *cobra.Command, args []string) { - exitCode = status.InspectSessionHistory - }, -} - -var debugOutputCmd = &cobra.Command{ - Use: "output", - Short: "shows output from last runs of resh", - Long: "Shows output from last runs of resh", - Run: func(cmd *cobra.Command, args []string) { - files := []string{ - "daemon_last_run_out.txt", - "collect_last_run_out.txt", - "postcollect_last_run_out.txt", - "session_init_last_run_out.txt", - "cli_last_run_out.txt", - } - dir := os.Getenv("__RESH_XDG_CACHE_HOME") - for _, fpath := range files { - fpath := filepath.Join(dir, fpath) - debugReadFile(fpath) - } - exitCode = status.Success - }, -} - -func debugReadFile(path string) { - fmt.Println("============================================================") - fmt.Println(" filepath:", path) - fmt.Println("============================================================") - dat, err := ioutil.ReadFile(path) - if err != nil { - fmt.Println("ERROR while reading file:", err) - } - fmt.Println(string(dat)) -} diff --git a/cmd/control/cmd/root.go b/cmd/control/cmd/root.go index 980d309..baf12b0 100644 --- a/cmd/control/cmd/root.go +++ b/cmd/control/cmd/root.go @@ -1,23 +1,20 @@ package cmd import ( - "fmt" - "log" - "os/user" - "path/filepath" - - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/cmd/control/status" - "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" "github.com/spf13/cobra" ) -// globals -var exitCode status.Code +// info passed during build var version string var commit string -var debug = false +var developement bool + +// globals var config cfg.Config +var out *output.Output var rootCmd = &cobra.Command{ Use: "reshctl", @@ -25,39 +22,28 @@ var rootCmd = &cobra.Command{ } // Execute reshctl -func Execute(ver, com string) status.Code { +func Execute(ver, com string) { version = ver commit = com - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, ".config/resh.toml") - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Println("Error reading config", err) - return status.Fail - } - if config.Debug { - debug = true - // log.SetFlags(log.LstdFlags | log.Lmicroseconds) + config, errCfg := cfg.New() + logger, _ := logger.New("reshctl", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + out = output.New(logger, "ERROR") + if errCfg != nil { + out.Error("Error while getting configuration", errCfg) } rootCmd.AddCommand(completionCmd) completionCmd.AddCommand(completionBashCmd) completionCmd.AddCommand(completionZshCmd) - rootCmd.AddCommand(debugCmd) - debugCmd.AddCommand(debugReloadCmd) - debugCmd.AddCommand(debugInspectCmd) - debugCmd.AddCommand(debugOutputCmd) - rootCmd.AddCommand(versionCmd) updateCmd.Flags().BoolVar(&betaFlag, "beta", false, "Update to latest version even if it's beta.") rootCmd.AddCommand(updateCmd) if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - return status.Fail + out.Fatal("Command ended with error", err) } - return exitCode } diff --git a/cmd/control/cmd/update.go b/cmd/control/cmd/update.go index dcdb21d..6263656 100644 --- a/cmd/control/cmd/update.go +++ b/cmd/control/cmd/update.go @@ -3,10 +3,8 @@ package cmd import ( "os" "os/exec" - "os/user" "path/filepath" - "github.com/curusarn/resh/cmd/control/status" "github.com/spf13/cobra" ) @@ -15,9 +13,11 @@ var updateCmd = &cobra.Command{ Use: "update", Short: "check for updates and update RESH", Run: func(cmd *cobra.Command, args []string) { - usr, _ := user.Current() - dir := usr.HomeDir - rawinstallPath := filepath.Join(dir, ".resh/rawinstall.sh") + homeDir, err := os.UserHomeDir() + if err != nil { + out.Fatal("Could not get user home dir", err) + } + rawinstallPath := filepath.Join(homeDir, ".resh/rawinstall.sh") execArgs := []string{rawinstallPath} if betaFlag { execArgs = append(execArgs, "--beta") @@ -25,9 +25,9 @@ var updateCmd = &cobra.Command{ execCmd := exec.Command("bash", execArgs...) execCmd.Stdout = os.Stdout execCmd.Stderr = os.Stderr - err := execCmd.Run() - if err == nil { - exitCode = status.Success + err = execCmd.Run() + if err != nil { + out.Fatal("Update ended with error", err) } }, } diff --git a/cmd/control/cmd/version.go b/cmd/control/cmd/version.go index 57c776f..9d86f20 100644 --- a/cmd/control/cmd/version.go +++ b/cmd/control/cmd/version.go @@ -4,13 +4,11 @@ import ( "encoding/json" "fmt" "io/ioutil" - "log" "net/http" "os" "strconv" - "github.com/curusarn/resh/cmd/control/status" - "github.com/curusarn/resh/pkg/msg" + "github.com/curusarn/resh/internal/msg" "github.com/spf13/cobra" ) @@ -24,13 +22,13 @@ var versionCmd = &cobra.Command{ commitEnv := getEnvVarWithDefault("__RESH_REVISION", "") printVersion("This terminal session", versionEnv, commitEnv) + // TODO: use output.Output.Error... for these resp, err := getDaemonStatus(config.Port) if err != nil { fmt.Fprintf(os.Stderr, "\nERROR: Resh-daemon didn't respond - it's probably not running.\n\n") fmt.Fprintf(os.Stderr, "-> Try restarting this terminal window to bring resh-daemon back up.\n") fmt.Fprintf(os.Stderr, "-> If the problem persists you can check resh-daemon logs: ~/.resh/daemon.log\n") fmt.Fprintf(os.Stderr, "-> You can file an issue at: https://github.com/curusarn/resh/issues\n") - exitCode = status.Fail return } printVersion("Currently running daemon", resp.Version, resp.Commit) @@ -48,7 +46,6 @@ var versionCmd = &cobra.Command{ return } - exitCode = status.ReshStatus }, } @@ -74,11 +71,11 @@ func getDaemonStatus(port int) (msg.StatusResponse, error) { defer resp.Body.Close() jsn, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Fatal("Error while reading 'daemon /status' response:", err) + out.Fatal("Error while reading 'daemon /status' response", err) } err = json.Unmarshal(jsn, &mess) if err != nil { - log.Fatal("Error while decoding 'daemon /status' response:", err) + out.Fatal("Error while decoding 'daemon /status' response", err) } return mess, nil } diff --git a/cmd/control/main.go b/cmd/control/main.go index ecae4e0..79d7289 100644 --- a/cmd/control/main.go +++ b/cmd/control/main.go @@ -1,8 +1,6 @@ package main import ( - "os" - "github.com/curusarn/resh/cmd/control/cmd" ) @@ -13,5 +11,5 @@ var version string var commit string func main() { - os.Exit(int(cmd.Execute(version, commit))) + cmd.Execute(version, commit) } diff --git a/cmd/control/status/status.go b/cmd/control/status/status.go deleted file mode 100644 index 0d797d7..0000000 --- a/cmd/control/status/status.go +++ /dev/null @@ -1,25 +0,0 @@ -package status - -// Code - exit code of the resh-control command -type Code int - -const ( - // Success exit code - Success Code = 0 - // Fail exit code - Fail = 1 - // EnableResh exit code - tells reshctl() wrapper to enable resh - // EnableResh = 30 - - // EnableControlRBinding exit code - tells reshctl() wrapper to enable control R binding - EnableControlRBinding = 32 - - // DisableControlRBinding exit code - tells reshctl() wrapper to disable control R binding - DisableControlRBinding = 42 - // ReloadRcFiles exit code - tells reshctl() wrapper to reload shellrc resh file - ReloadRcFiles = 50 - // InspectSessionHistory exit code - tells reshctl() wrapper to take current sessionID and send /inspect request to daemon - InspectSessionHistory = 51 - // ReshStatus exit code - tells reshctl() wrapper to show RESH status (aka systemctl status) - ReshStatus = 52 -) diff --git a/cmd/daemon/dump.go b/cmd/daemon/dump.go index 375da76..c038705 100644 --- a/cmd/daemon/dump.go +++ b/cmd/daemon/dump.go @@ -3,52 +3,49 @@ package main import ( "encoding/json" "io/ioutil" - "log" "net/http" - "github.com/curusarn/resh/pkg/histfile" - "github.com/curusarn/resh/pkg/msg" + "github.com/curusarn/resh/internal/histfile" + "github.com/curusarn/resh/internal/msg" + "go.uber.org/zap" ) type dumpHandler struct { + sugar *zap.SugaredLogger histfileBox *histfile.Histfile } func (h *dumpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if Debug { - log.Println("/dump START") - log.Println("/dump reading body ...") - } + sugar := h.sugar.With(zap.String("endpoint", "/dump")) + sugar.Debugw("Handling request, reading body ...") jsn, err := ioutil.ReadAll(r.Body) if err != nil { - log.Println("Error reading the body", err) + sugar.Errorw("Error reading body", "error", err) return } + sugar.Debugw("Unmarshaling record ...") mess := msg.CliMsg{} - if Debug { - log.Println("/dump unmarshaling record ...") - } err = json.Unmarshal(jsn, &mess) if err != nil { - log.Println("Decoding error:", err) - log.Println("Payload:", jsn) + sugar.Errorw("Error during unmarshaling", + "error", err, + "payload", jsn, + ) return } - if Debug { - log.Println("/dump dumping ...") - } + sugar.Debugw("Getting records to send ...") fullRecords := h.histfileBox.DumpCliRecords() if err != nil { - log.Println("Dump error:", err) + sugar.Errorw("Error when getting records", "error", err) } resp := msg.CliResponse{CliRecords: fullRecords.List} jsn, err = json.Marshal(&resp) if err != nil { - log.Println("Encoding error:", err) + sugar.Errorw("Error when marshaling", "error", err) return } w.Write(jsn) - log.Println("/dump END") + sugar.Infow("Request handled") } diff --git a/cmd/daemon/kill.go b/cmd/daemon/kill.go deleted file mode 100644 index 7c403a9..0000000 --- a/cmd/daemon/kill.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "io/ioutil" - "log" - "os/exec" - "strconv" - "strings" -) - -func killDaemon(pidfile string) error { - dat, err := ioutil.ReadFile(pidfile) - if err != nil { - log.Println("Reading pid file failed", err) - } - log.Print(string(dat)) - pid, err := strconv.Atoi(strings.TrimSuffix(string(dat), "\n")) - if err != nil { - log.Fatal("Pidfile contents are malformed", err) - } - cmd := exec.Command("kill", "-s", "sigint", strconv.Itoa(pid)) - err = cmd.Run() - if err != nil { - log.Printf("Command finished with error: %v", err) - return err - } - return nil -} diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index fc7b5bd..9338d0b 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -1,87 +1,141 @@ package main import ( - //"flag" - + "fmt" "io/ioutil" - "log" "os" - "os/user" + "os/exec" "path/filepath" "strconv" + "strings" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/httpclient" + "github.com/curusarn/resh/internal/logger" + "go.uber.org/zap" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string - -// Debug switch -var Debug = false +var developement bool func main() { - log.Println("Daemon starting... \n" + - "version: " + version + - " commit: " + commit) - usr, _ := user.Current() - dir := usr.HomeDir - pidfilePath := filepath.Join(dir, ".resh/resh.pid") - configPath := filepath.Join(dir, ".config/resh.toml") - reshHistoryPath := filepath.Join(dir, ".resh_history.json") - bashHistoryPath := filepath.Join(dir, ".bash_history") - zshHistoryPath := filepath.Join(dir, ".zsh_history") - logPath := filepath.Join(dir, ".resh/daemon.log") - - f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - log.Fatalf("Error opening file: %v\n", err) + config, errCfg := cfg.New() + logger, _ := logger.New("daemon", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) } - defer f.Close() + sugar := logger.Sugar() + d := daemon{sugar: sugar} + sugar.Infow("Deamon starting ...", + "version", version, + "commit", commit, + ) - log.SetOutput(f) - log.SetPrefix(strconv.Itoa(os.Getpid()) + " | ") + // xdgCacheHome := d.getEnvOrPanic("__RESH_XDG_CACHE_HOME") + // xdgDataHome := d.getEnvOrPanic("__RESH_XDG_DATA_HOME") - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Printf("Error reading config: %v\n", err) - return - } - if config.Debug { - Debug = true - log.SetFlags(log.LstdFlags | log.Lmicroseconds) + // TODO: rethink PID file and logs location + homeDir, err := os.UserHomeDir() + if err != nil { + sugar.Fatalw("Could not get user home dir", zap.Error(err)) } + PIDFile := filepath.Join(homeDir, ".resh/resh.pid") + reshHistoryPath := filepath.Join(homeDir, ".resh_history.json") + bashHistoryPath := filepath.Join(homeDir, ".bash_history") + zshHistoryPath := filepath.Join(homeDir, ".zsh_history") + + sugar = sugar.With(zap.Int("daemonPID", os.Getpid())) - res, err := isDaemonRunning(config.Port) + res, err := d.isDaemonRunning(config.Port) if err != nil { - log.Printf("Error while checking if the daemon is runnnig"+ - " - it's probably not running: %v\n", err) + sugar.Errorw("Error while checking daemon status - "+ + "it's probably not running", "error", err) } if res { - log.Println("Daemon is already running - exiting!") + sugar.Errorw("Daemon is already running - exiting!") return } - _, err = os.Stat(pidfilePath) + _, err = os.Stat(PIDFile) if err == nil { - log.Println("Pidfile exists") + sugar.Warn("Pidfile exists") // kill daemon - err = killDaemon(pidfilePath) + err = d.killDaemon(PIDFile) if err != nil { - log.Printf("Error while killing daemon: %v\n", err) + sugar.Errorw("Could not kill daemon", + "error", err, + ) } } - err = ioutil.WriteFile(pidfilePath, []byte(strconv.Itoa(os.Getpid())), 0644) + err = ioutil.WriteFile(PIDFile, []byte(strconv.Itoa(os.Getpid())), 0644) + if err != nil { + sugar.Fatalw("Could not create pidfile", + "error", err, + "PIDFile", PIDFile, + ) + } + server := Server{ + sugar: sugar, + config: config, + reshHistoryPath: reshHistoryPath, + bashHistoryPath: bashHistoryPath, + zshHistoryPath: zshHistoryPath, + } + server.Run() + sugar.Infow("Removing PID file ...", + "PIDFile", PIDFile, + ) + err = os.Remove(PIDFile) + if err != nil { + sugar.Errorw("Could not delete PID file", "error", err) + } + sugar.Info("Shutting down ...") +} + +type daemon struct { + sugar *zap.SugaredLogger +} + +func (d *daemon) getEnvOrPanic(envVar string) string { + val, found := os.LookupEnv(envVar) + if !found { + d.sugar.Fatalw("Required env variable is not set", + "variableName", envVar, + ) + } + return val +} + +func (d *daemon) isDaemonRunning(port int) (bool, error) { + url := "http://localhost:" + strconv.Itoa(port) + "/status" + client := httpclient.New() + resp, err := client.Get(url) + if err != nil { + return false, err + } + defer resp.Body.Close() + return true, nil +} + +func (d *daemon) killDaemon(pidfile string) error { + dat, err := ioutil.ReadFile(pidfile) + if err != nil { + d.sugar.Errorw("Reading pid file failed", + "PIDFile", pidfile, + "error", err) + } + d.sugar.Infow("Succesfully read PID file", "contents", string(dat)) + pid, err := strconv.Atoi(strings.TrimSuffix(string(dat), "\n")) if err != nil { - log.Fatalf("Could not create pidfile: %v\n", err) + return fmt.Errorf("could not parse PID file contents: %w", err) } - runServer(config, reshHistoryPath, bashHistoryPath, zshHistoryPath) - log.Println("main: Removing pidfile ...") - err = os.Remove(pidfilePath) + d.sugar.Infow("Successfully parsed PID", "PID", pid) + cmd := exec.Command("kill", "-s", "sigint", strconv.Itoa(pid)) + err = cmd.Run() if err != nil { - log.Printf("Could not delete pidfile: %v\n", err) + return fmt.Errorf("kill command finished with error: %w", err) } - log.Println("main: Shutdown - bye") + return nil } diff --git a/cmd/daemon/record.go b/cmd/daemon/record.go index 38dce4b..e9acfcc 100644 --- a/cmd/daemon/record.go +++ b/cmd/daemon/record.go @@ -3,45 +3,58 @@ package main import ( "encoding/json" "io/ioutil" - "log" "net/http" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" ) +func NewRecordHandler(sugar *zap.SugaredLogger, subscribers []chan records.Record) recordHandler { + return recordHandler{ + sugar: sugar.With(zap.String("endpoint", "/record")), + subscribers: subscribers, + } +} + type recordHandler struct { + sugar *zap.SugaredLogger subscribers []chan records.Record } func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sugar := h.sugar.With(zap.String("endpoint", "/record")) + sugar.Debugw("Handling request, sending response, reading body ...") w.Write([]byte("OK\n")) jsn, err := ioutil.ReadAll(r.Body) // run rest of the handler as goroutine to prevent any hangups go func() { if err != nil { - log.Println("Error reading the body", err) + sugar.Errorw("Error reading body", "error", err) return } + sugar.Debugw("Unmarshaling record ...") record := records.Record{} err = json.Unmarshal(jsn, &record) if err != nil { - log.Println("Decoding error: ", err) - log.Println("Payload: ", jsn) + sugar.Errorw("Error during unmarshaling", + "error", err, + "payload", jsn, + ) return } part := "2" if record.PartOne { part = "1" } - log.Println("/record - ", record.CmdLine, " - part", part) + sugar := sugar.With( + "cmdLine", record.CmdLine, + "part", part, + ) + sugar.Debugw("Got record, sending to subscribers ...") for _, sub := range h.subscribers { sub <- record } + sugar.Debugw("Record sent to subscribers") }() - - // fmt.Println("cmd:", r.CmdLine) - // fmt.Println("pwd:", r.Pwd) - // fmt.Println("git:", r.GitWorkTree) - // fmt.Println("exit_code:", r.ExitCode) } diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index 9ad1310..f161672 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -5,14 +5,26 @@ import ( "os" "strconv" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/histfile" - "github.com/curusarn/resh/pkg/records" - "github.com/curusarn/resh/pkg/sesswatch" - "github.com/curusarn/resh/pkg/signalhandler" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/histfile" + "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/sesswatch" + "github.com/curusarn/resh/internal/signalhandler" + "go.uber.org/zap" ) -func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPath string) { +// TODO: turn server and handlers into package + +type Server struct { + sugar *zap.SugaredLogger + config cfg.Config + + reshHistoryPath string + bashHistoryPath string + zshHistoryPath string +} + +func (s *Server) Run() { var recordSubscribers []chan records.Record var sessionInitSubscribers []chan records.Record var sessionDropSubscribers []chan string @@ -29,8 +41,8 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa signalSubscribers = append(signalSubscribers, histfileSignals) maxHistSize := 10000 // lines minHistSizeKB := 2000 // roughly lines - histfileBox := histfile.New(histfileRecords, histfileSessionsToDrop, - reshHistoryPath, bashHistoryPath, zshHistoryPath, + histfileBox := histfile.New(s.sugar, histfileRecords, histfileSessionsToDrop, + s.reshHistoryPath, s.bashHistoryPath, s.zshHistoryPath, maxHistSize, minHistSizeKB, histfileSignals, shutdown) @@ -39,21 +51,27 @@ func runServer(config cfg.Config, reshHistoryPath, bashHistoryPath, zshHistoryPa recordSubscribers = append(recordSubscribers, sesswatchRecords) sesswatchSessionsToWatch := make(chan records.Record) sessionInitSubscribers = append(sessionInitSubscribers, sesswatchSessionsToWatch) - sesswatch.Go(sesswatchSessionsToWatch, sesswatchRecords, sessionDropSubscribers, config.SesswatchPeriodSeconds) + sesswatch.Go( + s.sugar, + sesswatchSessionsToWatch, + sesswatchRecords, + sessionDropSubscribers, + s.config.SesswatchPeriodSeconds, + ) // handlers mux := http.NewServeMux() - mux.HandleFunc("/status", statusHandler) - mux.Handle("/record", &recordHandler{subscribers: recordSubscribers}) - mux.Handle("/session_init", &sessionInitHandler{subscribers: sessionInitSubscribers}) - mux.Handle("/dump", &dumpHandler{histfileBox: histfileBox}) + mux.Handle("/status", &statusHandler{sugar: s.sugar}) + mux.Handle("/record", &recordHandler{sugar: s.sugar, subscribers: recordSubscribers}) + mux.Handle("/session_init", &sessionInitHandler{sugar: s.sugar, subscribers: sessionInitSubscribers}) + mux.Handle("/dump", &dumpHandler{sugar: s.sugar, histfileBox: histfileBox}) server := &http.Server{ - Addr: "localhost:" + strconv.Itoa(config.Port), + Addr: "localhost:" + strconv.Itoa(s.config.Port), Handler: mux, } go server.ListenAndServe() // signalhandler - takes over the main goroutine so when signal handler exists the whole program exits - signalhandler.Run(signalSubscribers, shutdown, server) + signalhandler.Run(s.sugar, signalSubscribers, shutdown, server) } diff --git a/cmd/daemon/session-init.go b/cmd/daemon/session-init.go index da1e236..2015fb0 100644 --- a/cmd/daemon/session-init.go +++ b/cmd/daemon/session-init.go @@ -3,36 +3,48 @@ package main import ( "encoding/json" "io/ioutil" - "log" "net/http" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" ) type sessionInitHandler struct { + sugar *zap.SugaredLogger subscribers []chan records.Record } func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sugar := h.sugar.With(zap.String("endpoint", "/session_init")) + sugar.Debugw("Handling request, sending response, reading body ...") w.Write([]byte("OK\n")) + // TODO: should we somehow check for errors here? jsn, err := ioutil.ReadAll(r.Body) // run rest of the handler as goroutine to prevent any hangups go func() { if err != nil { - log.Println("Error reading the body", err) + sugar.Errorw("Error reading body", "error", err) return } + sugar.Debugw("Unmarshaling record ...") record := records.Record{} err = json.Unmarshal(jsn, &record) if err != nil { - log.Println("Decoding error: ", err) - log.Println("Payload: ", jsn) + sugar.Errorw("Error during unmarshaling", + "error", err, + "payload", jsn, + ) return } - log.Println("/session_init - id:", record.SessionID, " - pid:", record.SessionPID) + sugar := sugar.With( + "sessionID", record.SessionID, + "sessionPID", record.SessionPID, + ) + sugar.Infow("Got session, sending to subscribers ...") for _, sub := range h.subscribers { sub <- record } + sugar.Debugw("Session sent to subscribers") }() } diff --git a/cmd/daemon/status.go b/cmd/daemon/status.go index 599b8dd..3c6b416 100644 --- a/cmd/daemon/status.go +++ b/cmd/daemon/status.go @@ -2,16 +2,19 @@ package main import ( "encoding/json" - "log" "net/http" - "strconv" - "github.com/curusarn/resh/pkg/httpclient" - "github.com/curusarn/resh/pkg/msg" + "github.com/curusarn/resh/internal/msg" + "go.uber.org/zap" ) -func statusHandler(w http.ResponseWriter, r *http.Request) { - log.Println("/status START") +type statusHandler struct { + sugar *zap.SugaredLogger +} + +func (h *statusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sugar := h.sugar.With(zap.String("endpoint", "/status")) + sugar.Debugw("Handling request ...") resp := msg.StatusResponse{ Status: true, Version: version, @@ -19,23 +22,12 @@ func statusHandler(w http.ResponseWriter, r *http.Request) { } jsn, err := json.Marshal(&resp) if err != nil { - log.Println("Encoding error:", err) - log.Println("Response:", resp) + sugar.Errorw("Error when marshaling", + "error", err, + "response", resp, + ) return } w.Write(jsn) - log.Println("/status END") -} - -func isDaemonRunning(port int) (bool, error) { - url := "http://localhost:" + strconv.Itoa(port) + "/status" - client := httpclient.New() - resp, err := client.Get(url) - if err != nil { - log.Printf("Error while checking daemon status - "+ - "it's probably not running: %v\n", err) - return false, err - } - defer resp.Body.Close() - return true, nil + sugar.Infow("Request handled") } diff --git a/cmd/postcollect/main.go b/cmd/postcollect/main.go index b5cb6b0..87f25b7 100644 --- a/cmd/postcollect/main.go +++ b/cmd/postcollect/main.go @@ -3,38 +3,42 @@ package main import ( "flag" "fmt" - "log" "os" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/collect" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" // "os/exec" - "os/user" + "path/filepath" "strconv" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string +var developement bool func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") + config, errCfg := cfg.New() + logger, _ := logger.New("postcollect", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) + } + out := output.New(logger, "resh-postcollect ERROR") + homeDir, err := os.UserHomeDir() + if err != nil { + out.Fatal("Could not get user home dir", err) + } + reshUUIDPath := filepath.Join(homeDir, "/.resh/resh-uuid") machineIDPath := "/etc/machine-id" - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) - } showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") @@ -92,24 +96,24 @@ func main() { } realtimeAfter, err := strconv.ParseFloat(*rta, 64) if err != nil { - log.Fatal("Flag Parsing error (rta):", err) + out.Fatal("Error while parsing flag --realtimeAfter", err) } realtimeBefore, err := strconv.ParseFloat(*rtb, 64) if err != nil { - log.Fatal("Flag Parsing error (rtb):", err) + out.Fatal("Error while parsing flag --realtimeBefore", err) } realtimeDuration := realtimeAfter - realtimeBefore - timezoneAfterOffset := collect.GetTimezoneOffsetInSeconds(*timezoneAfter) + timezoneAfterOffset := collect.GetTimezoneOffsetInSeconds(logger, *timezoneAfter) realtimeAfterLocal := realtimeAfter + timezoneAfterOffset realPwdAfter, err := filepath.EvalSymlinks(*pwdAfter) if err != nil { - log.Println("err while handling pwdAfter realpath:", err) + logger.Error("Error while handling pwdAfter realpath", zap.Error(err)) realPwdAfter = "" } - gitDirAfter, gitRealDirAfter := collect.GetGitDirs(*gitCdupAfter, *gitCdupExitCodeAfter, *pwdAfter) + gitDirAfter, gitRealDirAfter := collect.GetGitDirs(logger, *gitCdupAfter, *gitCdupExitCodeAfter, *pwdAfter) if *gitRemoteExitCodeAfter != 0 { *gitRemoteAfter = "" } @@ -150,5 +154,5 @@ func main() { ReshRevision: commit, }, } - collect.SendRecord(rec, strconv.Itoa(config.Port), "/record") + collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/record") } diff --git a/cmd/sanitize/main.go b/cmd/sanitize/main.go deleted file mode 100644 index c4fd60a..0000000 --- a/cmd/sanitize/main.go +++ /dev/null @@ -1,523 +0,0 @@ -package main - -import ( - "bufio" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "encoding/json" - "errors" - "flag" - "fmt" - "log" - "math" - "net/url" - "os" - "os/user" - "path" - "path/filepath" - "strconv" - "strings" - "unicode" - - "github.com/coreos/go-semver/semver" - "github.com/curusarn/resh/pkg/records" - giturls "github.com/whilp/git-urls" -) - -// version from git set during build -var version string - -// commit from git set during build -var commit string - -func main() { - usr, _ := user.Current() - dir := usr.HomeDir - historyPath := filepath.Join(dir, ".resh_history.json") - // outputPath := filepath.Join(dir, "resh_history_sanitized.json") - sanitizerDataPath := filepath.Join(dir, ".resh", "sanitizer_data") - - showVersion := flag.Bool("version", false, "Show version and exit") - showRevision := flag.Bool("revision", false, "Show git revision and exit") - trimHashes := flag.Int("trim-hashes", 12, "Trim hashes to N characters, '0' turns off trimming") - inputPath := flag.String("input", historyPath, "Input file") - outputPath := flag.String("output", "", "Output file (default: use stdout)") - - flag.Parse() - - if *showVersion == true { - fmt.Println(version) - os.Exit(0) - } - if *showRevision == true { - fmt.Println(commit) - os.Exit(0) - } - sanitizer := sanitizer{hashLength: *trimHashes} - err := sanitizer.init(sanitizerDataPath) - if err != nil { - log.Fatal("Sanitizer init() error:", err) - } - - inputFile, err := os.Open(*inputPath) - if err != nil { - log.Fatal("Open() resh history file error:", err) - } - defer inputFile.Close() - - var writer *bufio.Writer - if *outputPath == "" { - writer = bufio.NewWriter(os.Stdout) - } else { - outputFile, err := os.Create(*outputPath) - if err != nil { - log.Fatal("Create() output file error:", err) - } - defer outputFile.Close() - writer = bufio.NewWriter(outputFile) - } - defer writer.Flush() - - scanner := bufio.NewScanner(inputFile) - for scanner.Scan() { - record := records.Record{} - fallbackRecord := records.FallbackRecord{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) - } - record = records.Convert(&fallbackRecord) - } - err = sanitizer.sanitizeRecord(&record) - if err != nil { - log.Println("Line:", line) - log.Fatal("Sanitization error:", err) - } - outLine, err := json.Marshal(&record) - if err != nil { - log.Println("Line:", line) - log.Fatal("Encoding error:", err) - } - // fmt.Println(string(outLine)) - n, err := writer.WriteString(string(outLine) + "\n") - if err != nil { - log.Fatal(err) - } - if n == 0 { - log.Fatal("Nothing was written", n) - } - } -} - -type sanitizer struct { - hashLength int - whitelist map[string]bool -} - -func (s *sanitizer) init(dataPath string) error { - globalData := path.Join(dataPath, "whitelist.txt") - s.whitelist = loadData(globalData) - return nil -} - -func loadData(fname string) map[string]bool { - file, err := os.Open(fname) - if err != nil { - log.Fatal("Open() file error:", err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - data := make(map[string]bool) - for scanner.Scan() { - line := scanner.Text() - data[line] = true - } - return data -} - -func (s *sanitizer) sanitizeRecord(record *records.Record) error { - // hash directories of the paths - record.Pwd = s.sanitizePath(record.Pwd) - record.RealPwd = s.sanitizePath(record.RealPwd) - record.PwdAfter = s.sanitizePath(record.PwdAfter) - record.RealPwdAfter = s.sanitizePath(record.RealPwdAfter) - record.GitDir = s.sanitizePath(record.GitDir) - record.GitDirAfter = s.sanitizePath(record.GitDirAfter) - record.GitRealDir = s.sanitizePath(record.GitRealDir) - record.GitRealDirAfter = s.sanitizePath(record.GitRealDirAfter) - record.Home = s.sanitizePath(record.Home) - record.ShellEnv = s.sanitizePath(record.ShellEnv) - - // hash the most sensitive info, do not tokenize - record.Host = s.hashToken(record.Host) - record.Login = s.hashToken(record.Login) - record.MachineID = s.hashToken(record.MachineID) - - var err error - // this changes git url a bit but I'm still happy with the result - // e.g. "git@github.com:curusarn/resh" becomes "ssh://git@github.com/3385162f14d7/5a7b2909005c" - // notice the "ssh://" prefix - record.GitOriginRemote, err = s.sanitizeGitURL(record.GitOriginRemote) - if err != nil { - log.Println("Error while snitizing GitOriginRemote url", record.GitOriginRemote, ":", err) - return err - } - record.GitOriginRemoteAfter, err = s.sanitizeGitURL(record.GitOriginRemoteAfter) - if err != nil { - log.Println("Error while snitizing GitOriginRemoteAfter url", record.GitOriginRemoteAfter, ":", err) - return err - } - - // sanitization destroys original CmdLine length -> save it - record.CmdLength = len(record.CmdLine) - - record.CmdLine, err = s.sanitizeCmdLine(record.CmdLine) - if err != nil { - log.Fatal("Cmd:", record.CmdLine, "; sanitization error:", err) - } - record.RecallLastCmdLine, err = s.sanitizeCmdLine(record.RecallLastCmdLine) - if err != nil { - log.Fatal("RecallLastCmdLine:", record.RecallLastCmdLine, "; sanitization error:", err) - } - - if len(record.RecallActionsRaw) > 0 { - record.RecallActionsRaw, err = s.sanitizeRecallActions(record.RecallActionsRaw, record.ReshVersion) - if err != nil { - log.Println("RecallActionsRaw:", record.RecallActionsRaw, "; sanitization error:", err) - } - } - // add a flag to signify that the record has been sanitized - record.Sanitized = true - return nil -} - -func fixSeparator(str string) string { - if len(str) > 0 && str[0] == ';' { - return "|||" + str[1:] - } - return str -} - -func minIndex(str string, substrs []string) (idx, substrIdx int) { - minMatch := math.MaxInt32 - for i, sep := range substrs { - match := strings.Index(str, sep) - if match != -1 && match < minMatch { - minMatch = match - substrIdx = i - } - } - idx = minMatch - return -} - -// sanitizes the recall actions by replacing the recall prefix with it's length -func (s *sanitizer) sanitizeRecallActions(str string, reshVersion string) (string, error) { - if len(str) == 0 { - return "", nil - } - var separators []string - seps := []string{"|||"} - refVersion, err := semver.NewVersion("2.5.14") - if err != nil { - return str, fmt.Errorf("sanitizeRecallActions: semver error: %s", err.Error()) - } - if len(reshVersion) == 0 { - return str, errors.New("sanitizeRecallActions: record.ReshVersion is an empty string") - } - if reshVersion == "dev" { - reshVersion = "0.0.0" - } - if reshVersion[0] == 'v' { - reshVersion = reshVersion[1:] - } - recordVersion, err := semver.NewVersion(reshVersion) - if err != nil { - return str, fmt.Errorf("sanitizeRecallActions: semver error: %s; version string: %s", err.Error(), reshVersion) - } - if recordVersion.LessThan(*refVersion) { - seps = append(seps, ";") - } - - actions := []string{"arrow_up", "arrow_down", "control_R"} - for _, sep := range seps { - for _, action := range actions { - separators = append(separators, sep+action+":") - } - } - /* - - find any of {|||,;}{arrow_up,arrow_down,control_R}: in the recallActions (on the lowest index) - - use found substring to parse out the next prefix - - sanitize prefix - - add fixed substring and sanitized prefix to output - */ - doBreak := false - sanStr := "" - idx := 0 - var currSeparator string - tokenLen, sepIdx := minIndex(str, separators) - if tokenLen != 0 { - return str, errors.New("sanitizeReacallActions: unexpected string before first action/separator") - } - currSeparator = separators[sepIdx] - idx += len(currSeparator) - for !doBreak { - tokenLen, sepIdx := minIndex(str[idx:], separators) - if tokenLen > len(str[idx:]) { - tokenLen = len(str[idx:]) - doBreak = true - } - // token := str[idx : idx+tokenLen] - sanStr += fixSeparator(currSeparator) + strconv.Itoa(tokenLen) - currSeparator = separators[sepIdx] - idx += tokenLen + len(currSeparator) - } - return sanStr, nil -} - -func (s *sanitizer) sanitizeCmdLine(cmdLine string) (string, error) { - const optionEndingChars = "\"$'\\#[]!><|;{}()*,?~&=`:@^/+%." // all bash control characters, '=', ... - const optionAllowedChars = "-_" // characters commonly found inside of options - sanCmdLine := "" - buff := "" - - // simple options shouldn't be sanitized - // 1) whitespace 2) "-" or "--" 3) letters, digits, "-", "_" 4) ending whitespace or any of "=;)" - var optionDetected bool - - prevR3 := ' ' - prevR2 := ' ' - prevR := ' ' - for _, r := range cmdLine { - switch optionDetected { - case true: - if unicode.IsSpace(r) || strings.ContainsRune(optionEndingChars, r) { - // whitespace or option ends the option - // => add option unsanitized - optionDetected = false - if len(buff) > 0 { - sanCmdLine += buff - buff = "" - } - sanCmdLine += string(r) - } else if unicode.IsLetter(r) == false && unicode.IsDigit(r) == false && - strings.ContainsRune(optionAllowedChars, r) == false { - // r is not any of allowed chars for an option: letter, digit, "-" or "_" - // => sanitize - if len(buff) > 0 { - sanToken, err := s.sanitizeCmdToken(buff) - if err != nil { - log.Println("WARN: got error while sanitizing cmdLine:", cmdLine) - // return cmdLine, err - } - sanCmdLine += sanToken - buff = "" - } - sanCmdLine += string(r) - } else { - buff += string(r) - } - case false: - // split command on all non-letter and non-digit characters - if unicode.IsLetter(r) == false && unicode.IsDigit(r) == false { - // split token - if len(buff) > 0 { - sanToken, err := s.sanitizeCmdToken(buff) - if err != nil { - log.Println("WARN: got error while sanitizing cmdLine:", cmdLine) - // return cmdLine, err - } - sanCmdLine += sanToken - buff = "" - } - sanCmdLine += string(r) - } else { - if (unicode.IsSpace(prevR2) && prevR == '-') || - (unicode.IsSpace(prevR3) && prevR2 == '-' && prevR == '-') { - optionDetected = true - } - buff += string(r) - } - } - prevR3 = prevR2 - prevR2 = prevR - prevR = r - } - if len(buff) <= 0 { - // nothing in the buffer => work is done - return sanCmdLine, nil - } - if optionDetected { - // option detected => dont sanitize - sanCmdLine += buff - return sanCmdLine, nil - } - // sanitize - sanToken, err := s.sanitizeCmdToken(buff) - if err != nil { - log.Println("WARN: got error while sanitizing cmdLine:", cmdLine) - // return cmdLine, err - } - sanCmdLine += sanToken - return sanCmdLine, nil -} - -func (s *sanitizer) sanitizeGitURL(rawURL string) (string, error) { - if len(rawURL) <= 0 { - return rawURL, nil - } - parsedURL, err := giturls.Parse(rawURL) - if err != nil { - return rawURL, err - } - return s.sanitizeParsedURL(parsedURL) -} - -func (s *sanitizer) sanitizeURL(rawURL string) (string, error) { - if len(rawURL) <= 0 { - return rawURL, nil - } - parsedURL, err := url.Parse(rawURL) - if err != nil { - return rawURL, err - } - return s.sanitizeParsedURL(parsedURL) -} - -func (s *sanitizer) sanitizeParsedURL(parsedURL *url.URL) (string, error) { - parsedURL.Opaque = s.sanitizeToken(parsedURL.Opaque) - - userinfo := parsedURL.User.Username() // only get username => password won't even make it to the sanitized data - if len(userinfo) > 0 { - parsedURL.User = url.User(s.sanitizeToken(userinfo)) - } else { - // we need to do this because `gitUrls.Parse()` sets `User` to `url.User("")` instead of `nil` - parsedURL.User = nil - } - var err error - parsedURL.Host, err = s.sanitizeTwoPartToken(parsedURL.Host, ":") - if err != nil { - return parsedURL.String(), err - } - parsedURL.Path = s.sanitizePath(parsedURL.Path) - // ForceQuery bool - parsedURL.RawQuery = s.sanitizeToken(parsedURL.RawQuery) - parsedURL.Fragment = s.sanitizeToken(parsedURL.Fragment) - - return parsedURL.String(), nil -} - -func (s *sanitizer) sanitizePath(path string) string { - var sanPath string - for _, token := range strings.Split(path, "/") { - if s.whitelist[token] != true { - token = s.hashToken(token) - } - sanPath += token + "/" - } - if len(sanPath) > 0 { - sanPath = sanPath[:len(sanPath)-1] - } - return sanPath -} - -func (s *sanitizer) sanitizeTwoPartToken(token string, delimeter string) (string, error) { - tokenParts := strings.Split(token, delimeter) - if len(tokenParts) <= 1 { - return s.sanitizeToken(token), nil - } - if len(tokenParts) == 2 { - return s.sanitizeToken(tokenParts[0]) + delimeter + s.sanitizeToken(tokenParts[1]), nil - } - return token, errors.New("Token has more than two parts") -} - -func (s *sanitizer) sanitizeCmdToken(token string) (string, error) { - // there shouldn't be tokens with letters or digits mixed together with symbols - if len(token) <= 1 { - // NOTE: do not sanitize single letter tokens - return token, nil - } - if s.isInWhitelist(token) == true { - return token, nil - } - - isLettersOrDigits := true - // isDigits := true - isOtherCharacters := true - for _, r := range token { - if unicode.IsDigit(r) == false && unicode.IsLetter(r) == false { - isLettersOrDigits = false - // isDigits = false - } - // if unicode.IsDigit(r) == false { - // isDigits = false - // } - if unicode.IsDigit(r) || unicode.IsLetter(r) { - isOtherCharacters = false - } - } - // NOTE: I decided that I don't want a special sanitization for numbers - // if isDigits { - // return s.hashNumericToken(token), nil - // } - if isLettersOrDigits { - return s.hashToken(token), nil - } - if isOtherCharacters { - return token, nil - } - log.Println("WARN: cmd token is made of mix of letters or digits and other characters; token:", token) - // return token, errors.New("cmd token is made of mix of letters or digits and other characters") - return s.hashToken(token), errors.New("cmd token is made of mix of letters or digits and other characters") -} - -func (s *sanitizer) sanitizeToken(token string) string { - if len(token) <= 1 { - // NOTE: do not sanitize single letter tokens - return token - } - if s.isInWhitelist(token) { - return token - } - return s.hashToken(token) -} - -func (s *sanitizer) hashToken(token string) string { - if len(token) <= 0 { - return token - } - // hash with sha256 - sum := sha256.Sum256([]byte(token)) - return s.trimHash(hex.EncodeToString(sum[:])) -} - -func (s *sanitizer) hashNumericToken(token string) string { - if len(token) <= 0 { - return token - } - sum := sha256.Sum256([]byte(token)) - sumInt := int(binary.LittleEndian.Uint64(sum[:])) - if sumInt < 0 { - return strconv.Itoa(sumInt * -1) - } - return s.trimHash(strconv.Itoa(sumInt)) -} - -func (s *sanitizer) trimHash(hash string) string { - length := s.hashLength - if length <= 0 || len(hash) < length { - length = len(hash) - } - return hash[:length] -} - -func (s *sanitizer) isInWhitelist(token string) bool { - return s.whitelist[strings.ToLower(token)] == true -} diff --git a/cmd/session-init/main.go b/cmd/session-init/main.go index 75f5298..0205dec 100644 --- a/cmd/session-init/main.go +++ b/cmd/session-init/main.go @@ -3,37 +3,42 @@ package main import ( "flag" "fmt" - "log" "os" - "github.com/BurntSushi/toml" - "github.com/curusarn/resh/pkg/cfg" - "github.com/curusarn/resh/pkg/collect" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/collect" + "github.com/curusarn/resh/internal/logger" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" - "os/user" "path/filepath" "strconv" ) -// version from git set during build +// info passed during build var version string - -// commit from git set during build var commit string +var developement bool func main() { - usr, _ := user.Current() - dir := usr.HomeDir - configPath := filepath.Join(dir, "/.config/resh.toml") - reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") + config, errCfg := cfg.New() + logger, _ := logger.New("collect", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) + } + out := output.New(logger, "resh-collect ERROR") + + homeDir, err := os.UserHomeDir() + if err != nil { + out.Fatal("Could not get user home dir", err) + } + + reshUUIDPath := filepath.Join(homeDir, "/.resh/resh-uuid") machineIDPath := "/etc/machine-id" - var config cfg.Config - if _, err := toml.DecodeFile(configPath, &config); err != nil { - log.Fatal("Error reading config:", err) - } showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") @@ -106,20 +111,20 @@ func main() { } realtimeBefore, err := strconv.ParseFloat(*rtb, 64) if err != nil { - log.Fatal("Flag Parsing error (rtb):", err) + out.Fatal("Error while parsing flag --realtimeBefore", err) } realtimeSessionStart, err := strconv.ParseFloat(*rtsess, 64) if err != nil { - log.Fatal("Flag Parsing error (rt sess):", err) + out.Fatal("Error while parsing flag --realtimeSession", err) } realtimeSessSinceBoot, err := strconv.ParseFloat(*rtsessboot, 64) if err != nil { - log.Fatal("Flag Parsing error (rt sess boot):", err) + out.Fatal("Error while parsing flag --realtimeSessSinceBoot", err) } realtimeSinceSessionStart := realtimeBefore - realtimeSessionStart realtimeSinceBoot := realtimeSessSinceBoot + realtimeSinceSessionStart - timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(*timezoneBefore) + timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(logger, *timezoneBefore) realtimeBeforeLocal := realtimeBefore + timezoneBeforeOffset if *osReleaseID == "" { @@ -182,5 +187,5 @@ func main() { ReshRevision: commit, }, } - collect.SendRecord(rec, strconv.Itoa(config.Port), "/session_init") + collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/session_init") } diff --git a/conf/config.toml b/conf/config.toml index 9db8b94..52adf66 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -1,5 +1,5 @@ port = 2627 sesswatchPeriodSeconds = 120 sesshistInitHistorySize = 1000 -debug = false bindControlR = true +logVerbosity = info diff --git a/go.mod b/go.mod index ce2450a..4c3db28 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,27 @@ module github.com/curusarn/resh -go 1.16 +go 1.18 require ( github.com/BurntSushi/toml v0.4.1 github.com/awesome-gocui/gocui v1.0.0 - github.com/coreos/go-semver v0.3.0 - github.com/gdamore/tcell/v2 v2.4.0 // indirect - github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-shellwords v1.0.12 github.com/mitchellh/go-ps v1.0.0 github.com/spf13/cobra v1.2.1 - github.com/whilp/git-urls v1.0.0 + go.uber.org/zap v1.21.0 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 +) + +require ( + github.com/gdamore/encoding v1.0.0 // indirect + github.com/gdamore/tcell/v2 v2.4.0 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect golang.org/x/sys v0.0.0-20210903071746-97244b99971b // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index b44a8eb..c1e937d 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awesome-gocui/gocui v1.0.0 h1:1bf0DAr2JqWNxGFS8Kex4fM/khICjEnCi+a1+NfWy+w= github.com/awesome-gocui/gocui v1.0.0/go.mod h1:UvP3dP6+UsTGl9IuqP36wzz6Lemo90wn5p3tJvZ2OqY= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -57,11 +59,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -209,8 +211,10 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -239,10 +243,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= -github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -258,9 +261,15 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -405,7 +414,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b h1:3Dq0eVHn0uaQJmPO+/aYPI/fRMqdrVDbu7MQcku54gg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -475,6 +483,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -591,6 +600,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go new file mode 100644 index 0000000..4587bf1 --- /dev/null +++ b/internal/cfg/cfg.go @@ -0,0 +1,117 @@ +package cfg + +import ( + "fmt" + "os" + "path" + + "github.com/BurntSushi/toml" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// configFile used to parse the config file +type configFile struct { + Port *int + SesswatchPeriodSeconds *uint + SesshistInitHistorySize *int + + LogLevel *string + + BindControlR *bool + + // deprecated + BindArrowKeysBash *bool + BindArrowKeysZsh *bool + Debug *bool +} + +// Config returned by this package to be used in the rest of the project +type Config struct { + Port int + SesswatchPeriodSeconds uint + SesshistInitHistorySize int + LogLevel zapcore.Level + Debug bool + BindControlR bool +} + +// defaults for config +var defaults = Config{ + Port: 2627, + SesswatchPeriodSeconds: 120, + SesshistInitHistorySize: 1000, + LogLevel: zap.InfoLevel, + Debug: false, + BindControlR: true, +} + +func getConfigPath() (string, error) { + fname := "resh.toml" + xdgDir, found := os.LookupEnv("XDG_CONFIG_HOME") + if found { + return path.Join(xdgDir, fname), nil + } + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("could not get user home dir: %w", err) + } + return path.Join(homeDir, ".config", fname), nil +} + +func readConfig() (*configFile, error) { + var config configFile + path, err := getConfigPath() + if err != nil { + return &config, fmt.Errorf("could not get config file path: %w", err) + } + if _, err := toml.DecodeFile(path, &config); err != nil { + return &config, fmt.Errorf("could not decode config: %w", err) + } + return &config, nil +} + +func processAndFillDefaults(configF *configFile) (Config, error) { + config := defaults + + if configF.Port != nil { + config.Port = *configF.Port + } + if configF.SesswatchPeriodSeconds != nil { + config.SesswatchPeriodSeconds = *configF.SesswatchPeriodSeconds + } + if configF.SesshistInitHistorySize != nil { + config.SesshistInitHistorySize = *configF.SesshistInitHistorySize + } + + var err error + if configF.LogLevel != nil { + logLevel, err := zapcore.ParseLevel(*configF.LogLevel) + if err != nil { + err = fmt.Errorf("could not parse log level: %w", err) + } else { + config.LogLevel = logLevel + } + } + + if configF.BindControlR != nil { + config.BindControlR = *configF.BindControlR + } + + return config, err +} + +// New returns a config file +// returned config is always usable, returned errors are informative +func New() (Config, error) { + configF, err := readConfig() + if err != nil { + return defaults, fmt.Errorf("using default config because of error while getting config: %w", err) + } + + config, err := processAndFillDefaults(configF) + if err != nil { + return config, fmt.Errorf("errors while processing config: %w", err) + } + return config, nil +} diff --git a/pkg/collect/collect.go b/internal/collect/collect.go similarity index 59% rename from pkg/collect/collect.go rename to internal/collect/collect.go index b1c2fb6..179859a 100644 --- a/pkg/collect/collect.go +++ b/internal/collect/collect.go @@ -4,14 +4,15 @@ import ( "bytes" "encoding/json" "io/ioutil" - "log" "net/http" "path/filepath" "strconv" "strings" - "github.com/curusarn/resh/pkg/httpclient" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/httpclient" + "github.com/curusarn/resh/internal/output" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" ) // SingleResponse json struct @@ -21,23 +22,27 @@ type SingleResponse struct { } // SendRecord to daemon -func SendRecord(r records.Record, port, path string) { +func SendRecord(out *output.Output, r records.Record, port, path string) { + out.Logger.Debug("Sending record ...", + zap.String("cmdLine", r.CmdLine), + zap.String("sessionID", r.SessionID), + ) recJSON, err := json.Marshal(r) if err != nil { - log.Fatal("send err 1", err) + out.Fatal("Error while encoding record", err) } req, err := http.NewRequest("POST", "http://localhost:"+port+path, bytes.NewBuffer(recJSON)) if err != nil { - log.Fatal("send err 2", err) + out.Fatal("Error while sending record", err) } req.Header.Set("Content-Type", "application/json") client := httpclient.New() _, err = client.Do(req) if err != nil { - log.Fatal("resh-daemon is not running - try restarting this terminal") + out.FatalDaemonNotRunning(err) } } @@ -46,38 +51,38 @@ func ReadFileContent(path string) string { dat, err := ioutil.ReadFile(path) if err != nil { return "" - //log.Fatal("failed to open " + path) + //sugar.Fatal("failed to open " + path) } return strings.TrimSuffix(string(dat), "\n") } // GetGitDirs based on result of git "cdup" command -func GetGitDirs(cdup string, exitCode int, pwd string) (string, string) { +func GetGitDirs(logger *zap.Logger, cdup string, exitCode int, pwd string) (string, string) { if exitCode != 0 { return "", "" } abspath := filepath.Clean(filepath.Join(pwd, cdup)) realpath, err := filepath.EvalSymlinks(abspath) if err != nil { - log.Println("err while handling git dir paths:", err) + logger.Error("Error while handling git dir paths", zap.Error(err)) return "", "" } return abspath, realpath } // GetTimezoneOffsetInSeconds based on zone returned by date command -func GetTimezoneOffsetInSeconds(zone string) float64 { +func GetTimezoneOffsetInSeconds(logger *zap.Logger, zone string) float64 { // date +%z -> "+0200" hoursStr := zone[:3] minsStr := zone[3:] hours, err := strconv.Atoi(hoursStr) if err != nil { - log.Println("err while parsing hours in timezone offset:", err) + logger.Error("Error while parsing hours in timezone offset", zap.Error(err)) return -1 } mins, err := strconv.Atoi(minsStr) if err != nil { - log.Println("err while parsing mins in timezone offset:", err) + logger.Error("err while parsing minutes in timezone offset:", zap.Error(err)) return -1 } secs := ((hours * 60) + mins) * 60 diff --git a/pkg/histcli/histcli.go b/internal/histcli/histcli.go similarity index 92% rename from pkg/histcli/histcli.go rename to internal/histcli/histcli.go index f9b6611..9bedf61 100644 --- a/pkg/histcli/histcli.go +++ b/internal/histcli/histcli.go @@ -1,7 +1,7 @@ package histcli import ( - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/records" ) // Histcli is a dump of history preprocessed for resh cli purposes diff --git a/pkg/histfile/histfile.go b/internal/histfile/histfile.go similarity index 57% rename from pkg/histfile/histfile.go rename to internal/histfile/histfile.go index 17e7ffb..9d7cce0 100644 --- a/pkg/histfile/histfile.go +++ b/internal/histfile/histfile.go @@ -2,19 +2,21 @@ package histfile import ( "encoding/json" - "log" "math" "os" "strconv" "sync" - "github.com/curusarn/resh/pkg/histcli" - "github.com/curusarn/resh/pkg/histlist" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/histlist" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" ) // Histfile writes records to histfile type Histfile struct { + sugar *zap.SugaredLogger + sessionsMutex sync.Mutex sessions map[string]records.Record historyPath string @@ -31,16 +33,17 @@ type Histfile struct { } // New creates new histfile and runs its gorutines -func New(input chan records.Record, sessionsToDrop chan string, +func New(sugar *zap.SugaredLogger, input chan records.Record, sessionsToDrop chan string, reshHistoryPath string, bashHistoryPath string, zshHistoryPath string, maxInitHistSize int, minInitHistSizeKB int, signals chan os.Signal, shutdownDone chan string) *Histfile { hf := Histfile{ + sugar: sugar.With("module", "histfile"), sessions: map[string]records.Record{}, historyPath: reshHistoryPath, - bashCmdLines: histlist.New(), - zshCmdLines: histlist.New(), + bashCmdLines: histlist.New(sugar), + zshCmdLines: histlist.New(sugar), cliRecords: histcli.New(), } go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB) @@ -61,49 +64,58 @@ func (h *Histfile) loadCliRecords(recs []records.Record) { rec := recs[i] h.cliRecords.AddRecord(rec) } - log.Println("histfile: resh history loaded - history records count:", len(h.cliRecords.List)) + h.sugar.Infow("Resh history loaded", + "historyRecordsCount", len(h.cliRecords.List), + ) } // loadsHistory from resh_history and if there is not enough of it also load native shell histories func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHistSize, minInitHistSizeKB int) { h.recentMutex.Lock() defer h.recentMutex.Unlock() - log.Println("histfile: Checking if resh_history is large enough ...") + h.sugar.Infow("Checking if resh_history is large enough ...") fi, err := os.Stat(h.historyPath) var size int if err != nil { - log.Println("histfile ERROR: failed to stat resh_history file:", err) + h.sugar.Errorw("Failed to stat resh_history file", "error", err) } else { size = int(fi.Size()) } useNativeHistories := false if size/1024 < minInitHistSizeKB { useNativeHistories = true - log.Println("histfile WARN: resh_history is too small - loading native bash and zsh history ...") - h.bashCmdLines = records.LoadCmdLinesFromBashFile(bashHistoryPath) - log.Println("histfile: bash history loaded - cmdLine count:", len(h.bashCmdLines.List)) - h.zshCmdLines = records.LoadCmdLinesFromZshFile(zshHistoryPath) - log.Println("histfile: zsh history loaded - cmdLine count:", len(h.zshCmdLines.List)) + h.sugar.Warnw("Resh_history is too small - loading native bash and zsh history ...") + h.bashCmdLines = records.LoadCmdLinesFromBashFile(h.sugar, bashHistoryPath) + h.sugar.Infow("Bash history loaded", "cmdLineCount", len(h.bashCmdLines.List)) + h.zshCmdLines = records.LoadCmdLinesFromZshFile(h.sugar, zshHistoryPath) + h.sugar.Infow("Zsh history loaded", "cmdLineCount", len(h.zshCmdLines.List)) // no maxInitHistSize when using native histories maxInitHistSize = math.MaxInt32 } - log.Println("histfile: Loading resh history from file ...") - history := records.LoadFromFile(h.historyPath) - log.Println("histfile: resh history loaded from file - count:", len(history)) + h.sugar.Debugw("Loading resh history from file ...", + "historyFile", h.historyPath, + ) + history := records.LoadFromFile(h.sugar, h.historyPath) + h.sugar.Infow("Resh history loaded from file", + "historyFile", h.historyPath, + "recordCount", len(history), + ) go h.loadCliRecords(history) // NOTE: keeping this weird interface for now because we might use it in the future // when we only load bash or zsh history - reshCmdLines := loadCmdLines(history) - log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List)) + reshCmdLines := loadCmdLines(h.sugar, history) + h.sugar.Infow("Resh history loaded and processed", + "recordCount", len(reshCmdLines.List), + ) if useNativeHistories == false { h.bashCmdLines = reshCmdLines h.zshCmdLines = histlist.Copy(reshCmdLines) return } h.bashCmdLines.AddHistlist(reshCmdLines) - log.Println("histfile: bash history + resh history - cmdLine count:", len(h.bashCmdLines.List)) + h.sugar.Infow("Processed bash history and resh history together", "cmdLinecount", len(h.bashCmdLines.List)) h.zshCmdLines.AddHistlist(reshCmdLines) - log.Println("histfile: zsh history + resh history - cmdLine count:", len(h.zshCmdLines.List)) + h.sugar.Infow("Processed zsh history and resh history together", "cmdLineCount", len(h.zshCmdLines.List)) } // sessionGC reads sessionIDs from channel and deletes them from histfile struct @@ -111,15 +123,16 @@ func (h *Histfile) sessionGC(sessionsToDrop chan string) { for { func() { session := <-sessionsToDrop - log.Println("histfile: got session to drop", session) + sugar := h.sugar.With("sessionID", session) + sugar.Debugw("Got session to drop") h.sessionsMutex.Lock() defer h.sessionsMutex.Unlock() if part1, found := h.sessions[session]; found == true { - log.Println("histfile: Dropping session:", session) + sugar.Infow("Dropping session") delete(h.sessions, session) - go writeRecord(part1, h.historyPath) + go writeRecord(sugar, part1, h.historyPath) } else { - log.Println("histfile: No hanging parts for session:", session) + sugar.Infow("No hanging parts for session - nothing to drop") } }() } @@ -131,36 +144,56 @@ func (h *Histfile) writer(input chan records.Record, signals chan os.Signal, shu func() { select { case record := <-input: + part := "2" + if record.PartOne { + part = "1" + } + sugar := h.sugar.With( + "recordCmdLine", record.CmdLine, + "recordPart", part, + "recordShell", record.Shell, + ) + sugar.Debugw("Got record") h.sessionsMutex.Lock() defer h.sessionsMutex.Unlock() // allows nested sessions to merge records properly mergeID := record.SessionID + "_" + strconv.Itoa(record.Shlvl) + sugar = sugar.With("mergeID", mergeID) if record.PartOne { if _, found := h.sessions[mergeID]; found { - log.Println("histfile WARN: Got another first part of the records before merging the previous one - overwriting! " + - "(this happens in bash because bash-preexec runs when it's not supposed to)") + msg := "Got another first part of the records before merging the previous one - overwriting!" + if record.Shell == "zsh" { + sugar.Warnw(msg) + } else { + sugar.Infow(msg + " Unfortunately this is normal in bash, it can't be prevented.") + } } h.sessions[mergeID] = record } else { if part1, found := h.sessions[mergeID]; found == false { - log.Println("histfile ERROR: Got second part of records and nothing to merge it with - ignoring! (mergeID:", mergeID, ")") + sugar.Warnw("Got second part of record and nothing to merge it with - ignoring!") } else { delete(h.sessions, mergeID) - go h.mergeAndWriteRecord(part1, record) + go h.mergeAndWriteRecord(sugar, part1, record) } } case sig := <-signals: - log.Println("histfile: Got signal " + sig.String()) + sugar := h.sugar.With( + "signal", sig.String(), + ) + sugar.Infow("Got signal") h.sessionsMutex.Lock() defer h.sessionsMutex.Unlock() - log.Println("histfile DEBUG: Unlocked mutex") + sugar.Debugw("Unlocked mutex") for sessID, record := range h.sessions { - log.Printf("histfile WARN: Writing incomplete record for session: %v\n", sessID) - h.writeRecord(record) + sugar.Warnw("Writing incomplete record for session", + "sessionID", sessID, + ) + h.writeRecord(sugar, record) } - log.Println("histfile DEBUG: Shutdown success") + sugar.Debugw("Shutdown successful") shutdownDone <- "histfile" return } @@ -168,14 +201,14 @@ func (h *Histfile) writer(input chan records.Record, signals chan os.Signal, shu } } -func (h *Histfile) writeRecord(part1 records.Record) { - writeRecord(part1, h.historyPath) +func (h *Histfile) writeRecord(sugar *zap.SugaredLogger, part1 records.Record) { + writeRecord(sugar, part1, h.historyPath) } -func (h *Histfile) mergeAndWriteRecord(part1, part2 records.Record) { +func (h *Histfile) mergeAndWriteRecord(sugar *zap.SugaredLogger, part1, part2 records.Record) { err := part1.Merge(part2) if err != nil { - log.Println("Error while merging", err) + sugar.Errorw("Error while merging records", "error", err) return } @@ -189,57 +222,40 @@ func (h *Histfile) mergeAndWriteRecord(part1, part2 records.Record) { h.cliRecords.AddRecord(part1) }() - writeRecord(part1, h.historyPath) + writeRecord(sugar, part1, h.historyPath) } -func writeRecord(rec records.Record, outputPath string) { +func writeRecord(sugar *zap.SugaredLogger, rec records.Record, outputPath string) { recJSON, err := json.Marshal(rec) if err != nil { - log.Println("Marshalling error", err) + sugar.Errorw("Marshalling error", "error", err) return } f, err := os.OpenFile(outputPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - log.Println("Could not open file", err) + sugar.Errorw("Could not open file", "error", err) return } defer f.Close() _, err = f.Write(append(recJSON, []byte("\n")...)) if err != nil { - log.Printf("Error while writing: %v, %s\n", rec, err) + sugar.Errorw("Error while writing record", + "recordRaw", rec, + "error", err, + ) return } } -// GetRecentCmdLines returns recent cmdLines -func (h *Histfile) GetRecentCmdLines(shell string, limit int) histlist.Histlist { - // NOTE: limit does nothing atm - h.recentMutex.Lock() - defer h.recentMutex.Unlock() - log.Println("histfile: History requested ...") - var hl histlist.Histlist - if shell == "bash" { - hl = histlist.Copy(h.bashCmdLines) - log.Println("histfile: history copied (bash) - cmdLine count:", len(hl.List)) - return hl - } - if shell != "zsh" { - log.Println("histfile ERROR: Unknown shell: ", shell) - } - hl = histlist.Copy(h.zshCmdLines) - log.Println("histfile: history copied (zsh) - cmdLine count:", len(hl.List)) - return hl -} - // DumpCliRecords returns enriched records func (h *Histfile) DumpCliRecords() histcli.Histcli { // don't forget locks in the future return h.cliRecords } -func loadCmdLines(recs []records.Record) histlist.Histlist { - hl := histlist.New() +func loadCmdLines(sugar *zap.SugaredLogger, recs []records.Record) histlist.Histlist { + hl := histlist.New(sugar) // go from bottom and deduplicate var cmdLines []string cmdLinesSet := map[string]bool{} diff --git a/pkg/histlist/histlist.go b/internal/histlist/histlist.go similarity index 62% rename from pkg/histlist/histlist.go rename to internal/histlist/histlist.go index a3f4334..7c80c7f 100644 --- a/pkg/histlist/histlist.go +++ b/internal/histlist/histlist.go @@ -1,9 +1,11 @@ package histlist -import "log" +import "go.uber.org/zap" // Histlist is a deduplicated list of cmdLines type Histlist struct { + // TODO: I'm not excited about logger being passed here + sugar *zap.SugaredLogger // list of commands lines (deduplicated) List []string // lookup: cmdLine -> last index @@ -11,13 +13,16 @@ type Histlist struct { } // New Histlist -func New() Histlist { - return Histlist{LastIndex: make(map[string]int)} +func New(sugar *zap.SugaredLogger) Histlist { + return Histlist{ + sugar: sugar.With("component", "histlist"), + LastIndex: make(map[string]int), + } } // Copy Histlist func Copy(hl Histlist) Histlist { - newHl := New() + newHl := New(hl.sugar) // copy list newHl.List = make([]string, len(hl.List)) copy(newHl.List, hl.List) @@ -36,7 +41,10 @@ func (h *Histlist) AddCmdLine(cmdLine string) { if found { // remove duplicate if cmdLine != h.List[idx] { - log.Println("histlist ERROR: Adding cmdLine:", cmdLine, " != LastIndex[cmdLine]:", h.List[idx]) + h.sugar.DPanicw("Index key is different than actual cmd line in the list", + "indexKeyCmdLine", cmdLine, + "actualCmdLine", h.List[idx], + ) } h.List = append(h.List[:idx], h.List[idx+1:]...) // idx++ @@ -44,7 +52,10 @@ func (h *Histlist) AddCmdLine(cmdLine string) { cmdLn := h.List[idx] h.LastIndex[cmdLn]-- if idx != h.LastIndex[cmdLn] { - log.Println("histlist ERROR: Shifting LastIndex idx:", idx, " != LastIndex[cmdLn]:", h.LastIndex[cmdLn]) + h.sugar.DPanicw("Index position is different than actual position of the cmd line", + "actualPosition", idx, + "indexedPosition", h.LastIndex[cmdLn], + ) } idx++ } @@ -53,7 +64,10 @@ func (h *Histlist) AddCmdLine(cmdLine string) { h.LastIndex[cmdLine] = len(h.List) // append new cmdline h.List = append(h.List, cmdLine) - // log.Println("histlist: Added cmdLine:", cmdLine, "; history length:", lenBefore, "->", len(h.List)) + h.sugar.Debugw("Added cmdLine", + "cmdLine", cmdLine, + "historyLength", len(h.List), + ) } // AddHistlist contents of another histlist to this histlist diff --git a/pkg/httpclient/httpclient.go b/internal/httpclient/httpclient.go similarity index 100% rename from pkg/httpclient/httpclient.go rename to internal/httpclient/httpclient.go diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..b031b77 --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,28 @@ +package logger + +import ( + "fmt" + "os" + "path/filepath" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func New(executable string, level zapcore.Level, developement bool) (*zap.Logger, error) { + // TODO: consider getting log path from config ? + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("error while getting home dir: %w", err) + } + logPath := filepath.Join(homeDir, ".resh/log.json") + loggerConfig := zap.NewProductionConfig() + loggerConfig.OutputPaths = []string{logPath} + loggerConfig.Level.SetLevel(level) + loggerConfig.Development = developement // DPanic panics in developement + logger, err := loggerConfig.Build() + if err != nil { + return logger, fmt.Errorf("error while creating logger: %w", err) + } + return logger.With(zap.String("executable", executable)), err +} diff --git a/pkg/msg/msg.go b/internal/msg/msg.go similarity index 92% rename from pkg/msg/msg.go rename to internal/msg/msg.go index 3c85987..cf8e3ea 100644 --- a/pkg/msg/msg.go +++ b/internal/msg/msg.go @@ -1,6 +1,6 @@ package msg -import "github.com/curusarn/resh/pkg/records" +import "github.com/curusarn/resh/internal/records" // CliMsg struct type CliMsg struct { diff --git a/internal/output/output.go b/internal/output/output.go new file mode 100644 index 0000000..769f099 --- /dev/null +++ b/internal/output/output.go @@ -0,0 +1,69 @@ +package output + +import ( + "fmt" + "os" + + "go.uber.org/zap" +) + +// Output wrapper for writting to logger and stdout/stderr at the same time +// useful for errors that should be presented to the user +type Output struct { + Logger *zap.Logger + ErrPrefix string +} + +func New(logger *zap.Logger, prefix string) *Output { + return &Output{ + Logger: logger, + ErrPrefix: prefix, + } +} + +func (f *Output) Info(msg string) { + fmt.Fprintf(os.Stdout, msg) + f.Logger.Info(msg) +} + +func (f *Output) Error(msg string, err error) { + fmt.Fprintf(os.Stderr, "%s: %s: %v", f.ErrPrefix, msg, err) + f.Logger.Error(msg, zap.Error(err)) +} + +func (f *Output) Fatal(msg string, err error) { + fmt.Fprintf(os.Stderr, "%s: %s: %v", f.ErrPrefix, msg, err) + f.Logger.Fatal(msg, zap.Error(err)) +} + +var msgDeamonNotRunning = `Resh-daemon didn't respond - it's probably not running. + + -> Try restarting this terminal window to bring resh-daemon back up + -> If the problem persists you can check resh-daemon logs: ~/.resh/log.json + -> You can create an issue at: https://github.com/curusarn/resh/issues +` +var msgVersionMismatch = `This terminal session was started with different resh version than is installed now. +It looks like you updated resh and didn't restart this terminal. + + -> Restart this terminal window to fix that +` + +func (f *Output) ErrorDaemonNotRunning(err error) { + fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgDeamonNotRunning) + f.Logger.Error("Daemon is not running", zap.Error(err)) +} + +func (f *Output) FatalDaemonNotRunning(err error) { + fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgDeamonNotRunning) + f.Logger.Fatal("Daemon is not running", zap.Error(err)) +} + +func (f *Output) ErrorVersionMismatch(err error) { + fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgVersionMismatch) + f.Logger.Fatal("Version mismatch", zap.Error(err)) +} + +func (f *Output) FatalVersionMismatch(err error) { + fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgVersionMismatch) + f.Logger.Fatal("Version mismatch", zap.Error(err)) +} diff --git a/pkg/records/records.go b/internal/records/records.go similarity index 88% rename from pkg/records/records.go rename to internal/records/records.go index ef0f561..b1f9a85 100644 --- a/pkg/records/records.go +++ b/internal/records/records.go @@ -4,15 +4,16 @@ import ( "bufio" "encoding/json" "errors" + "fmt" "io" - "log" "math" "os" "strconv" "strings" - "github.com/curusarn/resh/pkg/histlist" + "github.com/curusarn/resh/internal/histlist" "github.com/mattn/go-shellwords" + "go.uber.org/zap" ) // BaseRecord - common base for Record and FallbackRecord @@ -219,14 +220,14 @@ func Enriched(r Record) EnrichedRecord { if err != nil { record.Errors = append(record.Errors, "Validate error:"+err.Error()) // rec, _ := record.ToString() - // log.Println("Invalid command:", rec) + // sugar.Println("Invalid command:", rec) record.Invalid = true } record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine) if err != nil { record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error()) // rec, _ := record.ToString() - // log.Println("Invalid command:", rec) + // sugar.Println("Invalid command:", rec) record.Invalid = true // should this be really invalid ? } return record @@ -327,7 +328,7 @@ func (r *EnrichedRecord) SetCmdLine(cmdLine string) { r.Command, r.FirstWord, err = GetCommandAndFirstWord(cmdLine) if err != nil { r.Errors = append(r.Errors, "GetCommandAndFirstWord error:"+err.Error()) - // log.Println("Invalid command:", r.CmdLine) + // sugar.Println("Invalid command:", r.CmdLine) r.Invalid = true } } @@ -352,7 +353,7 @@ func Stripped(r EnrichedRecord) EnrichedRecord { func GetCommandAndFirstWord(cmdLine string) (string, string, error) { args, err := shellwords.Parse(cmdLine) if err != nil { - // log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") + // Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") return "", "", err } if len(args) == 0 { @@ -361,15 +362,14 @@ func GetCommandAndFirstWord(cmdLine string) (string, string, error) { i := 0 for true { // commands in shell sometimes look like this `variable=something command argument otherArgument --option` - // to get the command we skip over tokens that contain '=' + // to get the command we skip over tokens that contain '=' if strings.ContainsRune(args[i], '=') && len(args) > i+1 { i++ continue } return args[i], args[0], nil } - log.Fatal("GetCommandAndFirstWord error: this should not happen!") - return "ERROR", "ERROR", errors.New("this should not happen - contact developer ;)") + return "ERROR", "ERROR", errors.New("failed to retrieve first word of command") } // NormalizeGitRemote func @@ -511,22 +511,19 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { } // LoadFromFile loads records from 'fname' file -func LoadFromFile(fname string) []Record { - const allowedErrors = 2 +func LoadFromFile(sugar *zap.SugaredLogger, fname string) []Record { + const allowedErrors = 3 var encounteredErrors int - // NOTE: limit does nothing atm var recs []Record file, err := os.Open(fname) if err != nil { - log.Println("Open() resh history file error:", err) - log.Println("WARN: Skipping reading resh history!") + sugar.Error("Failed to open resh history file - skipping reading resh history", zap.Error(err)) return recs } defer file.Close() reader := bufio.NewReader(file) var i int - var firstErrLine int for { var line string line, err = reader.ReadString('\n') @@ -540,14 +537,16 @@ func LoadFromFile(fname string) []Record { if err != nil { err = json.Unmarshal([]byte(line), &fallbackRecord) if err != nil { - if encounteredErrors == 0 { - firstErrLine = i - } encounteredErrors++ - log.Println("Line:", line) - log.Println("Decoding error:", err) + sugar.Error("Could not decode line in resh history file", + "lineContents", line, + "lineNumber", i, + zap.Error(err), + ) if encounteredErrors > allowedErrors { - log.Fatalf("Fatal: Encountered more than %d decoding errors (%d)", allowedErrors, encounteredErrors) + sugar.Fatal("Encountered too many errors during decoding - exiting", + "allowedErrors", allowedErrors, + ) } } record = Convert(&fallbackRecord) @@ -555,32 +554,43 @@ func LoadFromFile(fname string) []Record { recs = append(recs, record) } if err != io.EOF { - log.Println("records: error while loading file:", err) + sugar.Error("Error while loading file", zap.Error(err)) } - // log.Println("records: Loaded lines - count:", i) + sugar.Infow("Loaded resh history records", + "recordCount", len(recs), + ) if encounteredErrors > 0 { // fix errors in the history file - log.Printf("There were %d decoding errors, the first error happend on line %d/%d\n", encounteredErrors, firstErrLine, i) + sugar.Warnw("Some history records could not be decoded - fixing resh history file by dropping them", + "corruptedRecords", encounteredErrors, + ) fnameBak := fname + ".bak" - log.Printf("Backing up current history file to %s\n", fnameBak) + sugar.Infow("Backing up current corrupted history file", + "backupFilename", fnameBak, + ) err := copyFile(fname, fnameBak) if err != nil { - log.Fatalln("Failed to backup history file with decode errors") + sugar.Errorw("Failed to create a backup history file - aborting fixing history file", + "backupFilename", fnameBak, + zap.Error(err), + ) + return recs } - log.Println("Writing out a history file without errors ...") + sugar.Info("Writing resh history file without errors ...") err = writeHistory(fname, recs) if err != nil { - log.Fatalln("Fatal: Failed write out new history") + sugar.Errorw("Failed write fixed history file - aborting fixing history file", + "filename", fname, + zap.Error(err), + ) } } - log.Println("records: Loaded records - count:", len(recs)) return recs } func copyFile(source, dest string) error { from, err := os.Open(source) if err != nil { - // log.Println("Open() resh history file error:", err) return err } defer from.Close() @@ -588,14 +598,12 @@ func copyFile(source, dest string) error { // to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666) to, err := os.Create(dest) if err != nil { - // log.Println("Create() resh history backup error:", err) return err } defer to.Close() _, err = io.Copy(to, from) if err != nil { - // log.Println("Copy() resh history to backup error:", err) return err } return nil @@ -604,14 +612,13 @@ func copyFile(source, dest string) error { func writeHistory(fname string, history []Record) error { file, err := os.Create(fname) if err != nil { - // log.Println("Create() resh history error:", err) return err } defer file.Close() for _, rec := range history { jsn, err := json.Marshal(rec) if err != nil { - log.Fatalln("Encode error!") + return fmt.Errorf("failed to encode record: %w", err) } file.Write(append(jsn, []byte("\n")...)) } @@ -619,12 +626,11 @@ func writeHistory(fname string, history []Record) error { } // LoadCmdLinesFromZshFile loads cmdlines from zsh history file -func LoadCmdLinesFromZshFile(fname string) histlist.Histlist { - hl := histlist.New() +func LoadCmdLinesFromZshFile(sugar *zap.SugaredLogger, fname string) histlist.Histlist { + hl := histlist.New(sugar) file, err := os.Open(fname) if err != nil { - log.Println("Open() zsh history file error:", err) - log.Println("WARN: Skipping reading zsh history!") + sugar.Error("Failed to open zsh history file - skipping reading zsh history", zap.Error(err)) return hl } defer file.Close() @@ -656,12 +662,11 @@ func LoadCmdLinesFromZshFile(fname string) histlist.Histlist { } // LoadCmdLinesFromBashFile loads cmdlines from bash history file -func LoadCmdLinesFromBashFile(fname string) histlist.Histlist { - hl := histlist.New() +func LoadCmdLinesFromBashFile(sugar *zap.SugaredLogger, fname string) histlist.Histlist { + hl := histlist.New(sugar) file, err := os.Open(fname) if err != nil { - log.Println("Open() bash history file error:", err) - log.Println("WARN: Skipping reading bash history!") + sugar.Error("Failed to open bash history file - skipping reading bash history", zap.Error(err)) return hl } defer file.Close() diff --git a/pkg/records/records_test.go b/internal/records/records_test.go similarity index 96% rename from pkg/records/records_test.go rename to internal/records/records_test.go index 5ef3c55..35c6920 100644 --- a/pkg/records/records_test.go +++ b/internal/records/records_test.go @@ -11,7 +11,7 @@ import ( func GetTestRecords() []Record { file, err := os.Open("testdata/resh_history.json") if err != nil { - log.Fatal("Open() resh history file error:", err) + log.Fatalf("Failed to open resh history file: %v", err) } defer file.Close() @@ -22,8 +22,7 @@ func GetTestRecords() []Record { line := scanner.Text() err = json.Unmarshal([]byte(line), &record) if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) + log.Fatalf("Error decoding record: '%s'; err: %v", line, err) } recs = append(recs, record) } diff --git a/pkg/records/testdata/resh_history.json b/internal/records/testdata/resh_history.json similarity index 100% rename from pkg/records/testdata/resh_history.json rename to internal/records/testdata/resh_history.json diff --git a/pkg/searchapp/highlight.go b/internal/searchapp/highlight.go similarity index 100% rename from pkg/searchapp/highlight.go rename to internal/searchapp/highlight.go diff --git a/pkg/searchapp/item.go b/internal/searchapp/item.go similarity index 99% rename from pkg/searchapp/item.go rename to internal/searchapp/item.go index 6908c25..c015428 100644 --- a/pkg/searchapp/item.go +++ b/internal/searchapp/item.go @@ -2,13 +2,12 @@ package searchapp import ( "fmt" - "log" "math" "strconv" "strings" "time" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/records" "golang.org/x/exp/utf8string" ) @@ -228,9 +227,6 @@ func produceLocation(length int, host string, pwdTilde string, differentHost boo shrinkFactor := float64(length) / float64(totalLen) shrinkedHostLen := int(math.Ceil(float64(hostLen) * shrinkFactor)) - if debug { - log.Printf("shrinkFactor: %f\n", shrinkFactor) - } halfLocationLen := length/2 - colonLen newHostLen = minInt(hostLen, shrinkedHostLen, halfLocationLen) diff --git a/pkg/searchapp/item_test.go b/internal/searchapp/item_test.go similarity index 100% rename from pkg/searchapp/item_test.go rename to internal/searchapp/item_test.go diff --git a/pkg/searchapp/query.go b/internal/searchapp/query.go similarity index 80% rename from pkg/searchapp/query.go rename to internal/searchapp/query.go index 2b8227d..fc2870a 100644 --- a/pkg/searchapp/query.go +++ b/internal/searchapp/query.go @@ -1,7 +1,6 @@ package searchapp import ( - "log" "sort" "strings" ) @@ -37,26 +36,16 @@ func filterTerms(terms []string) []string { // NewQueryFromString . func NewQueryFromString(queryInput string, host string, pwd string, gitOriginRemote string, debug bool) Query { - if debug { - log.Println("QUERY input = <" + queryInput + ">") - } terms := strings.Fields(queryInput) var logStr string for _, term := range terms { logStr += " <" + term + ">" } - if debug { - log.Println("QUERY raw terms =" + logStr) - } terms = filterTerms(terms) logStr = "" for _, term := range terms { logStr += " <" + term + ">" } - if debug { - log.Println("QUERY filtered terms =" + logStr) - log.Println("QUERY pwd =" + pwd) - } sort.SliceStable(terms, func(i, j int) bool { return len(terms[i]) < len(terms[j]) }) return Query{ terms: terms, @@ -68,17 +57,11 @@ func NewQueryFromString(queryInput string, host string, pwd string, gitOriginRem // GetRawTermsFromString . func GetRawTermsFromString(queryInput string, debug bool) []string { - if debug { - log.Println("QUERY input = <" + queryInput + ">") - } terms := strings.Fields(queryInput) var logStr string for _, term := range terms { logStr += " <" + term + ">" } - if debug { - log.Println("QUERY raw terms =" + logStr) - } terms = filterTerms(terms) logStr = "" for _, term := range terms { diff --git a/internal/searchapp/test.go b/internal/searchapp/test.go new file mode 100644 index 0000000..36d55e4 --- /dev/null +++ b/internal/searchapp/test.go @@ -0,0 +1,22 @@ +package searchapp + +import ( + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/msg" + "github.com/curusarn/resh/internal/records" + "go.uber.org/zap" +) + +// LoadHistoryFromFile ... +func LoadHistoryFromFile(sugar *zap.SugaredLogger, historyPath string, numLines int) msg.CliResponse { + recs := records.LoadFromFile(sugar, historyPath) + if numLines != 0 && numLines < len(recs) { + recs = recs[:numLines] + } + cliRecords := histcli.New() + for i := len(recs) - 1; i >= 0; i-- { + rec := recs[i] + cliRecords.AddRecord(rec) + } + return msg.CliResponse{CliRecords: cliRecords.List} +} diff --git a/pkg/searchapp/time.go b/internal/searchapp/time.go similarity index 100% rename from pkg/searchapp/time.go rename to internal/searchapp/time.go diff --git a/pkg/sess/sess.go b/internal/sess/sess.go similarity index 100% rename from pkg/sess/sess.go rename to internal/sess/sess.go diff --git a/pkg/sesswatch/sesswatch.go b/internal/sesswatch/sesswatch.go similarity index 55% rename from pkg/sesswatch/sesswatch.go rename to internal/sesswatch/sesswatch.go index 3d786c5..e5a55ec 100644 --- a/pkg/sesswatch/sesswatch.go +++ b/internal/sesswatch/sesswatch.go @@ -1,15 +1,17 @@ package sesswatch import ( - "log" "sync" "time" - "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/internal/records" "github.com/mitchellh/go-ps" + "go.uber.org/zap" ) type sesswatch struct { + sugar *zap.SugaredLogger + sessionsToDrop []chan string sleepSeconds uint @@ -18,8 +20,16 @@ type sesswatch struct { } // Go runs the session watcher - watches sessions and sends -func Go(sessionsToWatch chan records.Record, sessionsToWatchRecords chan records.Record, sessionsToDrop []chan string, sleepSeconds uint) { - sw := sesswatch{sessionsToDrop: sessionsToDrop, sleepSeconds: sleepSeconds, watchedSessions: map[string]bool{}} +func Go(sugar *zap.SugaredLogger, + sessionsToWatch chan records.Record, sessionsToWatchRecords chan records.Record, + sessionsToDrop []chan string, sleepSeconds uint) { + + sw := sesswatch{ + sugar: sugar.With("module", "sesswatch"), + sessionsToDrop: sessionsToDrop, + sleepSeconds: sleepSeconds, + watchedSessions: map[string]bool{}, + } go sw.waiter(sessionsToWatch, sessionsToWatchRecords) } @@ -31,46 +41,54 @@ func (s *sesswatch) waiter(sessionsToWatch chan records.Record, sessionsToWatchR // normal way to start watching a session id := record.SessionID pid := record.SessionPID + sugar := s.sugar.With( + "sessionID", record.SessionID, + "sessionPID", record.SessionPID, + ) s.mutex.Lock() defer s.mutex.Unlock() if s.watchedSessions[id] == false { - log.Println("sesswatch: start watching NEW session - id:", id, "; pid:", pid) + sugar.Infow("Starting watching new session") s.watchedSessions[id] = true - go s.watcher(id, pid) + go s.watcher(sugar, id, pid) } case record := <-sessionsToWatchRecords: // additional safety - watch sessions that were never properly initialized id := record.SessionID pid := record.SessionPID + sugar := s.sugar.With( + "sessionID", record.SessionID, + "sessionPID", record.SessionPID, + ) s.mutex.Lock() defer s.mutex.Unlock() if s.watchedSessions[id] == false { - log.Println("sesswatch WARN: start watching NEW session (based on /record) - id:", id, "; pid:", pid) + sugar.Warnw("Starting watching new session based on '/record'") s.watchedSessions[id] = true - go s.watcher(id, pid) + go s.watcher(sugar, id, pid) } } }() } } -func (s *sesswatch) watcher(sessionID string, sessionPID int) { +func (s *sesswatch) watcher(sugar *zap.SugaredLogger, sessionID string, sessionPID int) { for { time.Sleep(time.Duration(s.sleepSeconds) * time.Second) proc, err := ps.FindProcess(sessionPID) if err != nil { - log.Println("sesswatch ERROR: error while finding process - pid:", sessionPID) + sugar.Errorw("Error while finding process", "error", err) } else if proc == nil { - log.Println("sesswatch: Dropping session - id:", sessionID, "; pid:", sessionPID) + sugar.Infow("Dropping session") func() { s.mutex.Lock() defer s.mutex.Unlock() s.watchedSessions[sessionID] = false }() for _, ch := range s.sessionsToDrop { - log.Println("sesswatch: sending 'drop session' message ...") + sugar.Debugw("Sending 'drop session' message ...") ch <- sessionID - log.Println("sesswatch: sending 'drop session' message DONE") + sugar.Debugw("Sending 'drop session' message DONE") } break } diff --git a/internal/signalhandler/signalhander.go b/internal/signalhandler/signalhander.go new file mode 100644 index 0000000..b8188d6 --- /dev/null +++ b/internal/signalhandler/signalhander.go @@ -0,0 +1,74 @@ +package signalhandler + +import ( + "context" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "go.uber.org/zap" +) + +func sendSignals(sugar *zap.SugaredLogger, sig os.Signal, subscribers []chan os.Signal, done chan string) { + for _, sub := range subscribers { + sub <- sig + } + sugar.Warnw("Sent shutdown signals to components") + chanCount := len(subscribers) + start := time.Now() + delay := time.Millisecond * 100 + timeout := time.Millisecond * 2000 + + for { + select { + case _ = <-done: + chanCount-- + if chanCount == 0 { + sugar.Warnw("All components shut down successfully") + return + } + default: + time.Sleep(delay) + } + if time.Since(start) > timeout { + sugar.Errorw("Timouted while waiting for proper shutdown", + "componentsStillUp", strconv.Itoa(chanCount), + "timeout", timeout.String(), + ) + return + } + } +} + +// Run catches and handles signals +func Run(sugar *zap.SugaredLogger, subscribers []chan os.Signal, done chan string, server *http.Server) { + sugar = sugar.With("module", "signalhandler") + signals := make(chan os.Signal, 1) + + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + var sig os.Signal + for { + sig := <-signals + sugarSig := sugar.With("signal", sig.String()) + sugarSig.Infow("Got signal") + if sig == syscall.SIGTERM { + // Shutdown daemon on SIGTERM + break + } + sugarSig.Warnw("Ignoring signal. Send SIGTERM to trigger shutdown.") + } + + sugar.Infow("Sending shutdown signals to components ...") + sendSignals(sugar, sig, subscribers, done) + + sugar.Infow("Shutting down the server ...") + if err := server.Shutdown(context.Background()); err != nil { + sugar.Errorw("Error while shuting down HTTP server", + "error", err, + ) + } +} diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go deleted file mode 100644 index 06cc44d..0000000 --- a/pkg/cfg/cfg.go +++ /dev/null @@ -1,12 +0,0 @@ -package cfg - -// Config struct -type Config struct { - Port int - SesswatchPeriodSeconds uint - SesshistInitHistorySize int - Debug bool - BindArrowKeysBash bool - BindArrowKeysZsh bool - BindControlR bool -} diff --git a/pkg/searchapp/test.go b/pkg/searchapp/test.go deleted file mode 100644 index 30b850f..0000000 --- a/pkg/searchapp/test.go +++ /dev/null @@ -1,21 +0,0 @@ -package searchapp - -import ( - "github.com/curusarn/resh/pkg/histcli" - "github.com/curusarn/resh/pkg/msg" - "github.com/curusarn/resh/pkg/records" -) - -// LoadHistoryFromFile ... -func LoadHistoryFromFile(historyPath string, numLines int) msg.CliResponse { - recs := records.LoadFromFile(historyPath) - if numLines != 0 && numLines < len(recs) { - recs = recs[:numLines] - } - cliRecords := histcli.New() - for i := len(recs) - 1; i >= 0; i-- { - rec := recs[i] - cliRecords.AddRecord(rec) - } - return msg.CliResponse{CliRecords: cliRecords.List} -} diff --git a/pkg/signalhandler/signalhander.go b/pkg/signalhandler/signalhander.go deleted file mode 100644 index 5d2233e..0000000 --- a/pkg/signalhandler/signalhander.go +++ /dev/null @@ -1,65 +0,0 @@ -package signalhandler - -import ( - "context" - "log" - "net/http" - "os" - "os/signal" - "strconv" - "syscall" - "time" -) - -func sendSignals(sig os.Signal, subscribers []chan os.Signal, done chan string) { - for _, sub := range subscribers { - sub <- sig - } - chanCount := len(subscribers) - start := time.Now() - delay := time.Millisecond * 100 - timeout := time.Millisecond * 2000 - - for { - select { - case _ = <-done: - chanCount-- - if chanCount == 0 { - log.Println("signalhandler: All components shut down successfully") - return - } - default: - time.Sleep(delay) - } - if time.Since(start) > timeout { - log.Println("signalhandler: Timouted while waiting for proper shutdown - " + strconv.Itoa(chanCount) + " boxes are up after " + timeout.String()) - return - } - } -} - -// Run catches and handles signals -func Run(subscribers []chan os.Signal, done chan string, server *http.Server) { - signals := make(chan os.Signal, 1) - - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - var sig os.Signal - for { - sig := <-signals - log.Printf("signalhandler: Got signal '%s'\n", sig.String()) - if sig == syscall.SIGTERM { - // Shutdown daemon on SIGTERM - break - } - log.Printf("signalhandler: Ignoring signal '%s'. Send SIGTERM to trigger shutdown.\n", sig.String()) - } - - log.Println("signalhandler: Sending shutdown signals to components") - sendSignals(sig, subscribers, done) - - log.Println("signalhandler: Shutting down the server") - if err := server.Shutdown(context.Background()); err != nil { - log.Printf("HTTP server Shutdown: %v", err) - } -} diff --git a/scripts/util.sh b/scripts/util.sh index 4746eb0..96a643a 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -177,6 +177,7 @@ __resh_set_xdg_home_paths() { fi mkdir -p "$__RESH_XDG_CONFIG_FILE" >/dev/null 2>/dev/null __RESH_XDG_CONFIG_FILE="$__RESH_XDG_CONFIG_FILE/resh.toml" + export __RESH_XDG_CONFIG_FILE if [ -z "${XDG_CACHE_HOME-}" ]; then @@ -194,4 +195,5 @@ __resh_set_xdg_home_paths() { __RESH_XDG_DATA_HOME="$XDG_DATA_HOME/resh" fi mkdir -p "$__RESH_XDG_DATA_HOME" >/dev/null 2>/dev/null + export __RESH_XDG_CONFIG_FILE } From f6af6246ea94982ffb1dc3c8e59fbf88a14d4e5a Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 3 May 2022 03:00:43 +0200 Subject: [PATCH 16/35] fix --- cmd/cli/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 9ea53ae..fbd0219 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -101,11 +101,12 @@ func runReshCli(out *output.Output, config cfg.Config) (string, int) { } layout := manager{ + out: out, + config: config, sessionID: *sessionID, host: *host, pwd: *pwd, gitOriginRemote: records.NormalizeGitRemote(*gitOriginRemote), - config: config, s: &st, } g.SetManager(layout) @@ -598,7 +599,7 @@ func SendCliMsg(out *output.Output, m msg.CliMsg, port string) msg.CliResponse { if err != nil { out.Fatal("Failed decode response", err) } - sugar.Debug("Recieved records from daemon", + sugar.Debugw("Recieved records from daemon", "recordCount", len(response.CliRecords), ) return response From 8aded88f69f2d9084c16920f9674c6807cfb0cf9 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 3 May 2022 03:22:40 +0200 Subject: [PATCH 17/35] hint sh shebang --- scripts/hooks.sh | 1 + scripts/reshctl.sh | 1 + scripts/shellrc.sh | 1 + scripts/util.sh | 2 ++ scripts/widgets.sh | 1 + 5 files changed, 6 insertions(+) diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 146ddf9..5d9ebe3 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -1,3 +1,4 @@ +#!/hint/sh __resh_reset_variables() { __RESH_RECORD_ID=$(__resh_get_uuid) diff --git a/scripts/reshctl.sh b/scripts/reshctl.sh index 31d1a0f..d043366 100644 --- a/scripts/reshctl.sh +++ b/scripts/reshctl.sh @@ -1,3 +1,4 @@ +#!/hint/sh # shellcheck source=../submodules/bash-zsh-compat-widgets/bindfunc.sh . ~/.resh/bindfunc.sh diff --git a/scripts/shellrc.sh b/scripts/shellrc.sh index c21f335..311ed2a 100644 --- a/scripts/shellrc.sh +++ b/scripts/shellrc.sh @@ -1,3 +1,4 @@ +#!/hint/sh PATH=$PATH:~/.resh/bin # if [ -n "$ZSH_VERSION" ]; then diff --git a/scripts/util.sh b/scripts/util.sh index 96a643a..f2e2754 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -1,3 +1,5 @@ +#!/hint/sh + # util.sh - resh utility functions __resh_get_uuid() { cat /proc/sys/kernel/random/uuid 2>/dev/null || resh-uuid diff --git a/scripts/widgets.sh b/scripts/widgets.sh index 26f5b07..fd0a0b9 100644 --- a/scripts/widgets.sh +++ b/scripts/widgets.sh @@ -1,3 +1,4 @@ +#!/hint/sh # shellcheck source=hooks.sh . ~/.resh/hooks.sh From 439e60dab8823112b31e978b8aeee36297ea9428 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 4 May 2022 01:32:18 +0200 Subject: [PATCH 18/35] add timeouts --- cmd/daemon/run-server.go | 9 +++++++-- internal/collect/collect.go | 6 ++++-- scripts/util.sh | 12 +----------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index f161672..7878f9a 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -4,6 +4,7 @@ import ( "net/http" "os" "strconv" + "time" "github.com/curusarn/resh/internal/cfg" "github.com/curusarn/resh/internal/histfile" @@ -67,8 +68,12 @@ func (s *Server) Run() { mux.Handle("/dump", &dumpHandler{sugar: s.sugar, histfileBox: histfileBox}) server := &http.Server{ - Addr: "localhost:" + strconv.Itoa(s.config.Port), - Handler: mux, + Addr: "localhost:" + strconv.Itoa(s.config.Port), + Handler: mux, + ReadTimeout: 1 * time.Second, + WriteTimeout: 1 * time.Second, + ReadHeaderTimeout: 1 * time.Second, + IdleTimeout: 30 * time.Second, } go server.ListenAndServe() diff --git a/internal/collect/collect.go b/internal/collect/collect.go index 179859a..a72825c 100644 --- a/internal/collect/collect.go +++ b/internal/collect/collect.go @@ -8,8 +8,8 @@ import ( "path/filepath" "strconv" "strings" + "time" - "github.com/curusarn/resh/internal/httpclient" "github.com/curusarn/resh/internal/output" "github.com/curusarn/resh/internal/records" "go.uber.org/zap" @@ -39,7 +39,9 @@ func SendRecord(out *output.Output, r records.Record, port, path string) { } req.Header.Set("Content-Type", "application/json") - client := httpclient.New() + client := http.Client{ + Timeout: 1 * time.Second, + } _, err = client.Do(req) if err != nil { out.FatalDaemonNotRunning(err) diff --git a/scripts/util.sh b/scripts/util.sh index f2e2754..b434ecc 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -172,16 +172,6 @@ __resh_session_init() { } __resh_set_xdg_home_paths() { - if [ -z "${XDG_CONFIG_HOME-}" ]; then - __RESH_XDG_CONFIG_FILE="$HOME/.config" - else - __RESH_XDG_CONFIG_FILE="$XDG_CONFIG_HOME" - fi - mkdir -p "$__RESH_XDG_CONFIG_FILE" >/dev/null 2>/dev/null - __RESH_XDG_CONFIG_FILE="$__RESH_XDG_CONFIG_FILE/resh.toml" - export __RESH_XDG_CONFIG_FILE - - if [ -z "${XDG_CACHE_HOME-}" ]; then __RESH_XDG_CACHE_HOME="$HOME/.cache/resh" else @@ -197,5 +187,5 @@ __resh_set_xdg_home_paths() { __RESH_XDG_DATA_HOME="$XDG_DATA_HOME/resh" fi mkdir -p "$__RESH_XDG_DATA_HOME" >/dev/null 2>/dev/null - export __RESH_XDG_CONFIG_FILE + export __RESH_XDG_DATA_HOME } From bafb2445888e649839d432cb54fd9aa9d296ccbb Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 4 May 2022 04:01:58 +0200 Subject: [PATCH 19/35] config migration during install --- .goreleaser.yml | 9 +++ Makefile | 2 +- cmd/collect/main.go | 4 +- cmd/config-setup/main.go | 46 +++++++++++++++ cmd/postcollect/main.go | 4 +- cmd/session-init/main.go | 4 +- internal/cfg/cfg.go | 61 +++++++++++++------- internal/cfg/migrate.go | 110 ++++++++++++++++++++++++++++++++++++ internal/collect/collect.go | 15 ++--- scripts/install.sh | 29 ++-------- 10 files changed, 224 insertions(+), 60 deletions(-) create mode 100644 cmd/config-setup/main.go create mode 100644 internal/cfg/migrate.go diff --git a/.goreleaser.yml b/.goreleaser.yml index 125b906..c2e2bf5 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -93,6 +93,15 @@ builds: - amd64 - arm - arm64 + - + id: "config-setup" + main: ./cmd/config-setup + binary: bin/resh-config-setup + goarch: + - 386 + - amd64 + - arm + - arm64 # signs: # - artifacts: checksum diff --git a/Makefile b/Makefile index b269c6d..9a30621 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.de build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect bin/resh-daemon\ - bin/resh-control bin/resh-config bin/resh-cli + bin/resh-control bin/resh-config bin/resh-cli bin/resh-config-setup install: build scripts/install.sh diff --git a/cmd/collect/main.go b/cmd/collect/main.go index a0a17b2..aac282c 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -207,7 +207,7 @@ func main() { GitDir: gitDir, GitRealDir: gitRealDir, GitOriginRemote: *gitRemote, - MachineID: collect.ReadFileContent(machineIDPath), + MachineID: collect.ReadFileContent(out.Logger, machineIDPath), OsReleaseID: *osReleaseID, OsReleaseVersionID: *osReleaseVersionID, @@ -217,7 +217,7 @@ func main() { PartOne: true, - ReshUUID: collect.ReadFileContent(reshUUIDPath), + ReshUUID: collect.ReadFileContent(out.Logger, reshUUIDPath), ReshVersion: version, ReshRevision: commit, }, diff --git a/cmd/config-setup/main.go b/cmd/config-setup/main.go new file mode 100644 index 0000000..a1ed942 --- /dev/null +++ b/cmd/config-setup/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + "os" + + "github.com/curusarn/resh/internal/cfg" + "github.com/curusarn/resh/internal/logger" + "go.uber.org/zap" +) + +// info passed during build +var version string +var commit string +var developement bool + +func main() { + errDo := doConfigSetup() + config, errCfg := cfg.New() + logger, _ := logger.New("config-setup", config.LogLevel, developement) + defer logger.Sync() // flushes buffer, if any + + if errDo != nil { + logger.Error("Config setup failed", zap.Error(errDo)) + // TODO: better error message for people + fmt.Fprintf(os.Stderr, "ERROR: %v\n", errDo) + } + if errCfg != nil { + logger.Error("Error while getting configuration", zap.Error(errCfg)) + } +} + +func doConfigSetup() error { + err := cfg.Touch() + if err != nil { + return fmt.Errorf("could not touch config file: %w", err) + } + changes, err := cfg.Migrate() + if err != nil { + return fmt.Errorf("could not migrate config file version: %v", err) + } + if changes { + fmt.Printf("Config file format has changed - your config was updated to reflect the changes.\n") + } + return nil +} diff --git a/cmd/postcollect/main.go b/cmd/postcollect/main.go index 87f25b7..4d4fca8 100644 --- a/cmd/postcollect/main.go +++ b/cmd/postcollect/main.go @@ -145,11 +145,11 @@ func main() { GitDirAfter: gitDirAfter, GitRealDirAfter: gitRealDirAfter, GitOriginRemoteAfter: *gitRemoteAfter, - MachineID: collect.ReadFileContent(machineIDPath), + MachineID: collect.ReadFileContent(out.Logger, machineIDPath), PartOne: false, - ReshUUID: collect.ReadFileContent(reshUUIDPath), + ReshUUID: collect.ReadFileContent(out.Logger, reshUUIDPath), ReshVersion: version, ReshRevision: commit, }, diff --git a/cmd/session-init/main.go b/cmd/session-init/main.go index 0205dec..59010f7 100644 --- a/cmd/session-init/main.go +++ b/cmd/session-init/main.go @@ -174,7 +174,7 @@ func main() { RealtimeSinceSessionStart: realtimeSinceSessionStart, RealtimeSinceBoot: realtimeSinceBoot, - MachineID: collect.ReadFileContent(machineIDPath), + MachineID: collect.ReadFileContent(out.Logger, machineIDPath), OsReleaseID: *osReleaseID, OsReleaseVersionID: *osReleaseVersionID, @@ -182,7 +182,7 @@ func main() { OsReleaseName: *osReleaseName, OsReleasePrettyName: *osReleasePrettyName, - ReshUUID: collect.ReadFileContent(reshUUIDPath), + ReshUUID: collect.ReadFileContent(out.Logger, reshUUIDPath), ReshVersion: version, ReshRevision: commit, }, diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 4587bf1..9c3ae7a 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -12,38 +12,55 @@ import ( // configFile used to parse the config file type configFile struct { + // ConfigVersion - never remove this + ConfigVersion *string + + // added in legacy Port *int SesswatchPeriodSeconds *uint SesshistInitHistorySize *int + BindControlR *bool + Debug *bool + // added in v1 LogLevel *string - BindControlR *bool - - // deprecated + // added in legacy + // deprecated in v1 BindArrowKeysBash *bool BindArrowKeysZsh *bool - Debug *bool } // Config returned by this package to be used in the rest of the project type Config struct { - Port int - SesswatchPeriodSeconds uint + // Port used by daemon and rest of the components to communicate + // Make sure to restart the daemon when you change it + Port int + + // BindControlR causes CTRL+R to launch the search app + BindControlR bool + // LogLevel used to filter logs + LogLevel zapcore.Level + + // Debug mode for search app + Debug bool + // SesswatchPeriodSeconds is how often should daemon check if terminal + // sessions are still alive + SesswatchPeriodSeconds uint + // SesshistInitHistorySize is how large resh history needs to be for + // daemon to ignore standard shell history files SesshistInitHistorySize int - LogLevel zapcore.Level - Debug bool - BindControlR bool } // defaults for config var defaults = Config{ - Port: 2627, + Port: 2627, + LogLevel: zap.InfoLevel, + BindControlR: true, + + Debug: false, SesswatchPeriodSeconds: 120, SesshistInitHistorySize: 1000, - LogLevel: zap.InfoLevel, - Debug: false, - BindControlR: true, } func getConfigPath() (string, error) { @@ -59,18 +76,22 @@ func getConfigPath() (string, error) { return path.Join(homeDir, ".config", fname), nil } -func readConfig() (*configFile, error) { +func readConfig(path string) (*configFile, error) { var config configFile - path, err := getConfigPath() - if err != nil { - return &config, fmt.Errorf("could not get config file path: %w", err) - } if _, err := toml.DecodeFile(path, &config); err != nil { return &config, fmt.Errorf("could not decode config: %w", err) } return &config, nil } +func getConfig() (*configFile, error) { + path, err := getConfigPath() + if err != nil { + return nil, fmt.Errorf("could not get config file path: %w", err) + } + return readConfig(path) +} + func processAndFillDefaults(configF *configFile) (Config, error) { config := defaults @@ -104,9 +125,9 @@ func processAndFillDefaults(configF *configFile) (Config, error) { // New returns a config file // returned config is always usable, returned errors are informative func New() (Config, error) { - configF, err := readConfig() + configF, err := getConfig() if err != nil { - return defaults, fmt.Errorf("using default config because of error while getting config: %w", err) + return defaults, fmt.Errorf("using default config because of error while getting/reading config: %w", err) } config, err := processAndFillDefaults(configF) diff --git a/internal/cfg/migrate.go b/internal/cfg/migrate.go new file mode 100644 index 0000000..4f91278 --- /dev/null +++ b/internal/cfg/migrate.go @@ -0,0 +1,110 @@ +package cfg + +import ( + "fmt" + "os" + + "github.com/BurntSushi/toml" +) + +// Touch config file +func Touch() error { + path, err := getConfigPath() + if err != nil { + return fmt.Errorf("could not get config file path: %w", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) + if err != nil { + return fmt.Errorf("could not open/create config file: %w", err) + } + err = file.Close() + if err != nil { + return fmt.Errorf("could not close config file: %w", err) + } + return nil +} + +// Migrate old config versions to current config version +// returns true if any changes were made to the config +func Migrate() (bool, error) { + path, err := getConfigPath() + if err != nil { + return false, fmt.Errorf("could not get config file path: %w", err) + } + configF, err := readConfig(path) + if err != nil { + return false, fmt.Errorf("could not read config: %w", err) + } + const current = "v1" + if configF.ConfigVersion != nil && *configF.ConfigVersion == current { + return false, nil + } + + if configF.ConfigVersion == nil { + configF, err = legacyToV1(configF) + if err != nil { + return true, fmt.Errorf("error converting config from version 'legacy' to 'v1': %w", err) + } + } + + if *configF.ConfigVersion != current { + return false, fmt.Errorf("unrecognized config version: '%s'", *configF.ConfigVersion) + } + err = writeConfig(configF, path) + if err != nil { + return true, fmt.Errorf("could not write migrated config: %w", err) + } + return true, nil +} + +func writeConfig(config *configFile, path string) error { + file, err := os.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0666) + if err != nil { + return fmt.Errorf("could not open config for writing: %w", err) + } + defer file.Close() + err = toml.NewEncoder(file).Encode(config) + if err != nil { + return fmt.Errorf("could not encode config: %w", err) + } + return nil +} + +func legacyToV1(config *configFile) (*configFile, error) { + if config.ConfigVersion != nil { + return nil, fmt.Errorf("config version is not 'legacy': '%s'", *config.ConfigVersion) + } + version := "v1" + newConf := configFile{ + ConfigVersion: &version, + } + // Remove defaults + if config.Port != nil && *config.Port != 2627 { + newConf.Port = config.Port + } + if config.SesswatchPeriodSeconds != nil && *config.SesswatchPeriodSeconds != 120 { + newConf.SesswatchPeriodSeconds = config.SesswatchPeriodSeconds + } + if config.SesshistInitHistorySize != nil && *config.SesshistInitHistorySize != 1000 { + newConf.SesshistInitHistorySize = config.SesshistInitHistorySize + } + if config.BindControlR != nil && *config.BindControlR != true { + newConf.BindControlR = config.BindControlR + } + if config.Debug != nil && *config.Debug != false { + newConf.Debug = config.Debug + } + return &newConf, nil +} + +// func v1ToV2(config *configFile) (*configFile, error) { +// if *config.ConfigVersion != "v1" { +// return nil, fmt.Errorf("config version is not 'legacy': '%s'", *config.ConfigVersion) +// } +// version := "v2" +// newConf := configFile{ +// ConfigVersion: &version, +// // Here goes all config fields - no need to prune defaults like we do for legacy +// } +// return &newConf, nil +// } diff --git a/internal/collect/collect.go b/internal/collect/collect.go index a72825c..e9fc585 100644 --- a/internal/collect/collect.go +++ b/internal/collect/collect.go @@ -15,12 +15,6 @@ import ( "go.uber.org/zap" ) -// SingleResponse json struct -type SingleResponse struct { - Found bool `json:"found"` - CmdLine string `json:"cmdline"` -} - // SendRecord to daemon func SendRecord(out *output.Output, r records.Record, port, path string) { out.Logger.Debug("Sending record ...", @@ -49,11 +43,14 @@ func SendRecord(out *output.Output, r records.Record, port, path string) { } // ReadFileContent and return it as a string -func ReadFileContent(path string) string { +func ReadFileContent(logger *zap.Logger, path string) string { dat, err := ioutil.ReadFile(path) if err != nil { + logger.Error("Error reading file", + zap.String("filePath", path), + zap.Error(err), + ) return "" - //sugar.Fatal("failed to open " + path) } return strings.TrimSuffix(string(dat), "\n") } @@ -84,7 +81,7 @@ func GetTimezoneOffsetInSeconds(logger *zap.Logger, zone string) float64 { } mins, err := strconv.Atoi(minsStr) if err != nil { - logger.Error("err while parsing minutes in timezone offset:", zap.Error(err)) + logger.Error("Errot while parsing minutes in timezone offset:", zap.Error(err)) return -1 } secs := ((hours * 60) + mins) * 60 diff --git a/scripts/install.sh b/scripts/install.sh index 5caedca..b07ac4d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -116,42 +116,23 @@ cp -f scripts/shellrc.sh ~/.resh/shellrc cp -f scripts/reshctl.sh scripts/widgets.sh scripts/hooks.sh scripts/util.sh ~/.resh/ cp -f scripts/rawinstall.sh ~/.resh/ -update_config() { - version=$1 - key=$2 - value=$3 - # TODO: create bin/semver-lt - if bin/semver-lt "${__RESH_VERSION:-0.0.0}" "$1" && [ "$(bin/resh-config -key $key)" != "$value" ] ; then - echo " * config option $key was updated to $value" - # TODO: enable resh-config value setting - # resh-config -key "$key" -value "$value" - fi -} - - -# Do not overwrite config if it exists -if [ ! -f ~/.config/resh.toml ]; then - echo "Copying config file ..." - cp -f conf/config.toml ~/.config/resh.toml -# else - # echo "Merging config files ..." - # NOTE: This is where we will merge configs when we make changes to the upstream config - # HINT: check which version are we updating FROM and make changes to config based on that -fi - echo "Generating completions ..." bin/resh-control completion bash > ~/.resh/bash_completion.d/_reshctl bin/resh-control completion zsh > ~/.resh/zsh_completion.d/_reshctl echo "Copying more files ..." cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid -cp -f bin/* ~/.resh/bin/ +rm ~/.resh/bin/resh-* ||: +cp -f bin/resh-{daemon,control,collect,postcollect,session-init,config} ~/.resh/bin/ cp -f scripts/resh-evaluate-plot.py ~/.resh/bin/ cp -fr data/sanitizer ~/.resh/sanitizer_data # backward compatibility: We have a new location for resh history file [ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json +echo "Checking config file ..." +./bin/resh-config-setup + echo "Finishing up ..." # Adding resh shellrc to .bashrc ... if [ ! -f ~/.bashrc ]; then From 9dfcb25d95470ae213df9dfbfc87bb92020d9710 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sun, 16 Oct 2022 20:07:59 +0200 Subject: [PATCH 20/35] checkpoint before continuing work after a while --- Makefile | 7 +- cmd/config-setup/main.go | 46 -- cmd/install-utils/backup.go | 5 + cmd/install-utils/main.go | 49 ++ cmd/install-utils/migrate.go | 36 + go.mod | 1 + go.sum | 2 + internal/cfg/cfg.go | 2 +- internal/datadir/datadir.go | 71 ++ internal/deviceid/deviceid.go | 18 + internal/histio/file.go | 56 ++ internal/histio/histio.go | 44 ++ internal/logger/logger.go | 9 +- internal/msg/msg.go | 19 +- internal/recconv/recconv.go | 9 + internal/recio/read.go | 158 ++++ internal/recio/recio.go | 13 + internal/recio/write.go | 46 ++ internal/recload/recload.go | 1 + internal/record/legacy.go | 88 +++ internal/record/record.go | 2 + internal/record/v1.go | 58 ++ internal/recordint/collect.go | 21 + internal/recordint/enriched.go | 51 ++ internal/recordint/flag.go | 9 + internal/recordint/indexed.go | 9 + internal/recordint/recordint.go | 2 + internal/recordint/searchapp.go | 40 + internal/records/records.go | 360 --------- internal/records/records_test.go | 66 -- internal/recutil/recutil.go | 94 +++ scripts/hooks.sh | 14 +- scripts/install.sh | 21 +- scripts/resh-evaluate-plot.py | 1218 ------------------------------ scripts/shellrc.sh | 17 +- scripts/util.sh | 35 +- 36 files changed, 914 insertions(+), 1783 deletions(-) delete mode 100644 cmd/config-setup/main.go create mode 100644 cmd/install-utils/backup.go create mode 100644 cmd/install-utils/main.go create mode 100644 cmd/install-utils/migrate.go create mode 100644 internal/datadir/datadir.go create mode 100644 internal/deviceid/deviceid.go create mode 100644 internal/histio/file.go create mode 100644 internal/histio/histio.go create mode 100644 internal/recconv/recconv.go create mode 100644 internal/recio/read.go create mode 100644 internal/recio/recio.go create mode 100644 internal/recio/write.go create mode 100644 internal/recload/recload.go create mode 100644 internal/record/legacy.go create mode 100644 internal/record/record.go create mode 100644 internal/record/v1.go create mode 100644 internal/recordint/collect.go create mode 100644 internal/recordint/enriched.go create mode 100644 internal/recordint/flag.go create mode 100644 internal/recordint/indexed.go create mode 100644 internal/recordint/recordint.go create mode 100644 internal/recordint/searchapp.go create mode 100644 internal/recutil/recutil.go delete mode 100755 scripts/resh-evaluate-plot.py diff --git a/Makefile b/Makefile index 9a30621..249cd43 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,9 @@ VERSION="${LATEST_TAG}-DEV" GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.development=true" -build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect bin/resh-daemon\ - bin/resh-control bin/resh-config bin/resh-cli bin/resh-config-setup +build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect\ + bin/resh-daemon bin/resh-control bin/resh-config bin/resh-cli\ + bin/installutil install: build scripts/install.sh @@ -21,7 +22,7 @@ rebuild: make build clean: - rm -f bin/resh-* + rm -f bin/* uninstall: # Uninstalling ... diff --git a/cmd/config-setup/main.go b/cmd/config-setup/main.go deleted file mode 100644 index a1ed942..0000000 --- a/cmd/config-setup/main.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/curusarn/resh/internal/cfg" - "github.com/curusarn/resh/internal/logger" - "go.uber.org/zap" -) - -// info passed during build -var version string -var commit string -var developement bool - -func main() { - errDo := doConfigSetup() - config, errCfg := cfg.New() - logger, _ := logger.New("config-setup", config.LogLevel, developement) - defer logger.Sync() // flushes buffer, if any - - if errDo != nil { - logger.Error("Config setup failed", zap.Error(errDo)) - // TODO: better error message for people - fmt.Fprintf(os.Stderr, "ERROR: %v\n", errDo) - } - if errCfg != nil { - logger.Error("Error while getting configuration", zap.Error(errCfg)) - } -} - -func doConfigSetup() error { - err := cfg.Touch() - if err != nil { - return fmt.Errorf("could not touch config file: %w", err) - } - changes, err := cfg.Migrate() - if err != nil { - return fmt.Errorf("could not migrate config file version: %v", err) - } - if changes { - fmt.Printf("Config file format has changed - your config was updated to reflect the changes.\n") - } - return nil -} diff --git a/cmd/install-utils/backup.go b/cmd/install-utils/backup.go new file mode 100644 index 0000000..9148164 --- /dev/null +++ b/cmd/install-utils/backup.go @@ -0,0 +1,5 @@ +package main + +func backup() { + +} diff --git a/cmd/install-utils/main.go b/cmd/install-utils/main.go new file mode 100644 index 0000000..4e83ec0 --- /dev/null +++ b/cmd/install-utils/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "flag" + "fmt" + "os" +) + +// info passed during build +var version string +var commit string +var developement bool + +func main() { + var command string + flag.StringVar(&command, "command", "", "Utility to run") + flag.Parse() + + switch command { + case "backup": + backup() + case "rollback": + rollback() + case "migrate-config": + migrateConfig() + case "migrate-history": + migrateHistory() + case "help": + printUsage(os.Stdout) + default: + fmt.Fprintf(os.Stderr, "ERROR: Unknown command") + printUsage(os.Stderr) + } +} + +func printUsage(f *os.File) { + usage := ` + Utils used during resh instalation + + USAGE: ./install-utils COMMAND + COMMANDS: + backup backup resh installation and data + rollback restore resh installation and data from backup + migrate-config update config to reflect updates + migrate-history update history to reflect updates + help show this help + ` + fmt.Fprintf(f, usage) +} diff --git a/cmd/install-utils/migrate.go b/cmd/install-utils/migrate.go new file mode 100644 index 0000000..e0bd34e --- /dev/null +++ b/cmd/install-utils/migrate.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + "os" + + "github.com/curusarn/resh/internal/cfg" +) + +func migrateConfig() { + err := cfg.Touch() + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to touch config file: %v\n", err) + os.Exit(1) + } + changes, err := cfg.Migrate() + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: Failed to update config file: %v\n", err) + os.Exit(1) + } + if changes { + fmt.Printf("Config file format has changed since last update - your config was updated to reflect the changes.\n") + } +} + +func migrateHistory() { + // homeDir, err := os.UserHomeDir() + // if err != nil { + + // } + + // TODO: Find history in: + // - xdg_data/resh/history.reshjson + // - .resh_history.json + // - .resh/history.json +} diff --git a/go.mod b/go.mod index 4c3db28..c63b38f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/mitchellh/go-ps v1.0.0 github.com/spf13/cobra v1.2.1 + github.com/whilp/git-urls v1.0.0 go.uber.org/zap v1.21.0 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 ) diff --git a/go.sum b/go.sum index c1e937d..2e9120d 100644 --- a/go.sum +++ b/go.sum @@ -246,6 +246,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 9c3ae7a..99f7c84 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -59,7 +59,7 @@ var defaults = Config{ BindControlR: true, Debug: false, - SesswatchPeriodSeconds: 120, + SesswatchPeriodSeconds: 600, SesshistInitHistorySize: 1000, } diff --git a/internal/datadir/datadir.go b/internal/datadir/datadir.go new file mode 100644 index 0000000..59011dd --- /dev/null +++ b/internal/datadir/datadir.go @@ -0,0 +1,71 @@ +package datadir + +import ( + "fmt" + "os" + "path" +) + +// You should not need this caching +// It messes with proper dependency injection +// Find another way + +// type dirCache struct { +// dir string +// err error +// +// cached bool +// } +// +// var cache dirCache +// +// func getPathNoCache() (string, error) { +// reshDir := "resh" +// xdgDir, found := os.LookupEnv("XDG_DATA_HOME") +// if found { +// return path.Join(xdgDir, reshDir), nil +// } +// homeDir, err := os.UserHomeDir() +// if err != nil { +// return "", fmt.Errorf("error while getting home dir: %w", err) +// } +// return path.Join(homeDir, ".local/share/", reshDir), nil +// } +// +// func GetPath() (string, error) { +// if !cache.cached { +// dir, err := getPathNoCache() +// cache = dirCache{ +// dir: dir, +// err: err, +// cached: true, +// } +// } +// return cache.dir, cache.err +// } + +func GetPath() (string, error) { + reshDir := "resh" + xdgDir, found := os.LookupEnv("XDG_DATA_HOME") + if found { + return path.Join(xdgDir, reshDir), nil + } + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("error while getting home dir: %w", err) + } + return path.Join(homeDir, ".local/share/", reshDir), nil +} + +func MakePath() (string, error) { + path, err := GetPath() + if err != nil { + return "", err + } + err = os.MkdirAll(path, 0755) + // skip "exists" error + if err != nil && !os.IsExist(err) { + return "", fmt.Errorf("error while creating directories: %w", err) + } + return path, nil +} diff --git a/internal/deviceid/deviceid.go b/internal/deviceid/deviceid.go new file mode 100644 index 0000000..79a080d --- /dev/null +++ b/internal/deviceid/deviceid.go @@ -0,0 +1,18 @@ +package deviceid + +import ( + "fmt" + "os" + "path" + "strings" +) + +func Get(dataDir string) (string, error) { + fname := "device-id" + dat, err := os.ReadFile(path.Join(dataDir, fname)) + if err != nil { + return "", fmt.Errorf("could not read file with device-id: %w", err) + } + id := strings.TrimRight(string(dat), "\n") + return id, nil +} diff --git a/internal/histio/file.go b/internal/histio/file.go new file mode 100644 index 0000000..5233717 --- /dev/null +++ b/internal/histio/file.go @@ -0,0 +1,56 @@ +package histio + +import ( + "fmt" + "os" + "sync" + + "github.com/curusarn/resh/internal/recio" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" +) + +type histfile struct { + sugar *zap.SugaredLogger + // deviceID string + path string + + mu sync.RWMutex + data []recordint.Indexed + fileinfo os.FileInfo +} + +func newHistfile(sugar *zap.SugaredLogger, path string) *histfile { + return &histfile{ + sugar: sugar.With( + // FIXME: drop V1 once original histfile is gone + "component", "histfileV1", + "path", path, + ), + // deviceID: deviceID, + path: path, + } +} + +func (h *histfile) updateFromFile() error { + rio := recio.New(h.sugar) + // TODO: decide and handle errors + newData, _, err := rio.ReadFile(h.path) + if err != nil { + return fmt.Errorf("could not read history file: %w", err) + } + h.mu.Lock() + defer h.mu.Unlock() + h.data = newData + h.updateFileInfo() + return nil +} + +func (h *histfile) updateFileInfo() error { + info, err := os.Stat(h.path) + if err != nil { + return fmt.Errorf("history file not found: %w", err) + } + h.fileinfo = info + return nil +} diff --git a/internal/histio/histio.go b/internal/histio/histio.go new file mode 100644 index 0000000..3bc7c68 --- /dev/null +++ b/internal/histio/histio.go @@ -0,0 +1,44 @@ +package histio + +import ( + "path" + + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" +) + +type Histio struct { + sugar *zap.SugaredLogger + histDir string + + thisDeviceID string + thisHistory *histfile + // TODO: remote histories + // moreHistories map[string]*histfile + + recordsToAppend chan record.V1 + recordsToFlag chan recordint.Flag +} + +func New(sugar *zap.SugaredLogger, dataDir, deviceID string) *Histio { + sugarHistio := sugar.With(zap.String("component", "histio")) + histDir := path.Join(dataDir, "history") + currPath := path.Join(histDir, deviceID) + // TODO: file extenstion for the history, yes or no? (.reshjson vs. ) + + // TODO: discover other history files, exclude current + + return &Histio{ + sugar: sugarHistio, + histDir: histDir, + + thisDeviceID: deviceID, + thisHistory: newHistfile(sugar, currPath), + // moreHistories: ... + } +} + +func (h *Histio) Append(r *record.V1) { + +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go index b031b77..3167412 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -2,20 +2,19 @@ package logger import ( "fmt" - "os" "path/filepath" + "github.com/curusarn/resh/internal/datadir" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) func New(executable string, level zapcore.Level, developement bool) (*zap.Logger, error) { - // TODO: consider getting log path from config ? - homeDir, err := os.UserHomeDir() + dataDir, err := datadir.GetPath() if err != nil { - return nil, fmt.Errorf("error while getting home dir: %w", err) + return nil, fmt.Errorf("error while getting resh data dir: %w", err) } - logPath := filepath.Join(homeDir, ".resh/log.json") + logPath := filepath.Join(dataDir, "log.json") loggerConfig := zap.NewProductionConfig() loggerConfig.OutputPaths = []string{logPath} loggerConfig.Level.SetLevel(level) diff --git a/internal/msg/msg.go b/internal/msg/msg.go index cf8e3ea..06d87cf 100644 --- a/internal/msg/msg.go +++ b/internal/msg/msg.go @@ -1,27 +1,14 @@ package msg -import "github.com/curusarn/resh/internal/records" - // CliMsg struct type CliMsg struct { - SessionID string `json:"sessionID"` - PWD string `json:"pwd"` + SessionID string + PWD string } // CliResponse struct type CliResponse struct { - CliRecords []records.CliRecord `json:"cliRecords"` -} - -// InspectMsg struct -type InspectMsg struct { - SessionID string `json:"sessionId"` - Count uint `json:"count"` -} - -// MultiResponse struct -type MultiResponse struct { - CmdLines []string `json:"cmdlines"` + Records []record.SearchApp } // StatusResponse struct diff --git a/internal/recconv/recconv.go b/internal/recconv/recconv.go new file mode 100644 index 0000000..d0156c5 --- /dev/null +++ b/internal/recconv/recconv.go @@ -0,0 +1,9 @@ +package recconv + +import "github.com/curusarn/resh/internal/record" + +func LegacyToV1(r *record.Legacy) *record.V1 { + return &record.V1{ + // FIXME: fill in all the fields + } +} diff --git a/internal/recio/read.go b/internal/recio/read.go new file mode 100644 index 0000000..7a7f183 --- /dev/null +++ b/internal/recio/read.go @@ -0,0 +1,158 @@ +package recio + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/curusarn/resh/internal/recconv" + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" +) + +func (r *RecIO) ReadAndFixFile(fpath string, maxErrors int) ([]recordint.Indexed, error) { + recs, numErrs, err := r.ReadFile(fpath) + if err != nil { + return nil, err + } + if numErrs > maxErrors { + return nil, fmt.Errorf("encountered too many decoding errors") + } + if numErrs == 0 { + return recs, nil + } + + // TODO: check there error messages + r.sugar.Warnw("Some history records could not be decoded - fixing resh history file by dropping them", + "corruptedRecords", numErrs, + ) + fpathBak := fpath + ".bak" + r.sugar.Infow("Backing up current corrupted history file", + "backupFilename", fpathBak, + ) + // TODO: maybe use upstram copy function + err = copyFile(fpath, fpathBak) + if err != nil { + r.sugar.Errorw("Failed to create a backup history file - aborting fixing history file", + "backupFilename", fpathBak, + zap.Error(err), + ) + return recs, nil + } + r.sugar.Info("Writing resh history file without errors ...") + var recsV1 []record.V1 + for _, rec := range recs { + recsV1 = append(recsV1, rec.Rec) + } + err = r.WriteFile(fpath, recsV1) + if err != nil { + r.sugar.Errorw("Failed write fixed history file - aborting fixing history file", + "filename", fpath, + zap.Error(err), + ) + } + return recs, nil +} + +func (r *RecIO) ReadFile(fpath string) ([]recordint.Indexed, int, error) { + var recs []recordint.Indexed + file, err := os.Open(fpath) + if err != nil { + return nil, 0, fmt.Errorf("failed to open history file: %w", err) + } + defer file.Close() + + reader := bufio.NewReader(file) + numErrs := 0 + var idx int + for { + var line string + line, err = reader.ReadString('\n') + if err != nil { + break + } + idx++ + rec, err := r.decodeLine(line) + if err != nil { + numErrs++ + continue + } + recidx := recordint.Indexed{ + Rec: *rec, + // TODO: Is line index actually enough? + // Don't we want to count bytes because we will scan by number of bytes? + // hint: https://benjamincongdon.me/blog/2018/04/10/Counting-Scanned-Bytes-in-Go/ + Idx: idx, + } + recs = append(recs, recidx) + } + if err != io.EOF { + r.sugar.Error("Error while loading file", zap.Error(err)) + } + r.sugar.Infow("Loaded resh history records", + "recordCount", len(recs), + ) + return recs, numErrs, nil +} + +func copyFile(source, dest string) error { + from, err := os.Open(source) + if err != nil { + return err + } + defer from.Close() + + // This is equivalnet to: os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666) + to, err := os.Create(dest) + if err != nil { + return err + } + defer to.Close() + + _, err = io.Copy(to, from) + if err != nil { + return err + } + return nil +} + +func (r *RecIO) decodeLine(line string) (*record.V1, error) { + idx := strings.Index(line, "{") + if idx == -1 { + return nil, fmt.Errorf("no openning brace found") + } + schema := line[:idx] + jsn := line[idx:] + switch schema { + case "v1": + var rec record.V1 + err := decodeAnyRecord(jsn, &rec) + if err != nil { + return nil, err + } + return &rec, nil + case "": + var rec record.Legacy + err := decodeAnyRecord(jsn, &rec) + if err != nil { + return nil, err + } + return recconv.LegacyToV1(&rec), nil + default: + return nil, fmt.Errorf("unknown record schema/type '%s'", schema) + } +} + +// TODO: find out if we are loosing performance because of the use of interface{} + +func decodeAnyRecord(jsn string, rec interface{}) error { + err := json.Unmarshal([]byte(jsn), &rec) + if err != nil { + return fmt.Errorf("failed to decode json: %w", err) + } + return nil +} diff --git a/internal/recio/recio.go b/internal/recio/recio.go new file mode 100644 index 0000000..5ea986b --- /dev/null +++ b/internal/recio/recio.go @@ -0,0 +1,13 @@ +package recio + +import ( + "go.uber.org/zap" +) + +type RecIO struct { + sugar *zap.SugaredLogger +} + +func New(sugar *zap.SugaredLogger) RecIO { + return RecIO{sugar: sugar} +} diff --git a/internal/recio/write.go b/internal/recio/write.go new file mode 100644 index 0000000..8bc5e06 --- /dev/null +++ b/internal/recio/write.go @@ -0,0 +1,46 @@ +package recio + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recordint" +) + +// TODO: better errors +func (r *RecIO) WriteFile(fpath string, data []record.V1) error { + file, err := os.Create(fpath) + if err != nil { + return err + } + defer file.Close() + for _, rec := range data { + jsn, err := encodeV1Record(rec) + if err != nil { + return err + } + _, err = file.Write(jsn) + if err != nil { + return err + } + } + return nil +} + +func (r *RecIO) EditRecordFlagsInFile(fpath string, idx int, rec recordint.Flag) error { + // FIXME: implement + // open file "not as append" + // scan to the correct line + + return nil +} + +func encodeV1Record(rec record.V1) ([]byte, error) { + jsn, err := json.Marshal(rec) + if err != nil { + return nil, fmt.Errorf("failed to encode json: %w", err) + } + return append(jsn, []byte("\n")...), nil +} diff --git a/internal/recload/recload.go b/internal/recload/recload.go new file mode 100644 index 0000000..0db3e3c --- /dev/null +++ b/internal/recload/recload.go @@ -0,0 +1 @@ +package recload diff --git a/internal/record/legacy.go b/internal/record/legacy.go new file mode 100644 index 0000000..3b913fb --- /dev/null +++ b/internal/record/legacy.go @@ -0,0 +1,88 @@ +package record + +type Legacy struct { + // core + CmdLine string `json:"cmdLine"` + ExitCode int `json:"exitCode"` + Shell string `json:"shell"` + Uname string `json:"uname"` + SessionID string `json:"sessionId"` + RecordID string `json:"recordId"` + + // posix + Home string `json:"home"` + Lang string `json:"lang"` + LcAll string `json:"lcAll"` + Login string `json:"login"` + Pwd string `json:"pwd"` + PwdAfter string `json:"pwdAfter"` + ShellEnv string `json:"shellEnv"` + Term string `json:"term"` + + // non-posix"` + RealPwd string `json:"realPwd"` + RealPwdAfter string `json:"realPwdAfter"` + Pid int `json:"pid"` + SessionPID int `json:"sessionPid"` + Host string `json:"host"` + Hosttype string `json:"hosttype"` + Ostype string `json:"ostype"` + Machtype string `json:"machtype"` + Shlvl int `json:"shlvl"` + + // before after + TimezoneBefore string `json:"timezoneBefore"` + TimezoneAfter string `json:"timezoneAfter"` + + RealtimeBefore float64 `json:"realtimeBefore"` + RealtimeAfter float64 `json:"realtimeAfter"` + RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` + RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` + + RealtimeDuration float64 `json:"realtimeDuration"` + RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` + RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` + + GitDir string `json:"gitDir"` + GitRealDir string `json:"gitRealDir"` + GitOriginRemote string `json:"gitOriginRemote"` + GitDirAfter string `json:"gitDirAfter"` + GitRealDirAfter string `json:"gitRealDirAfter"` + GitOriginRemoteAfter string `json:"gitOriginRemoteAfter"` + MachineID string `json:"machineId"` + + OsReleaseID string `json:"osReleaseId"` + OsReleaseVersionID string `json:"osReleaseVersionId"` + OsReleaseIDLike string `json:"osReleaseIdLike"` + OsReleaseName string `json:"osReleaseName"` + OsReleasePrettyName string `json:"osReleasePrettyName"` + + ReshUUID string `json:"reshUuid"` + ReshVersion string `json:"reshVersion"` + ReshRevision string `json:"reshRevision"` + + // records come in two parts (collect and postcollect) + PartOne bool `json:"partOne,omitempty"` // false => part two + PartsMerged bool `json:"partsMerged"` + // special flag -> not an actual record but an session end + SessionExit bool `json:"sessionExit,omitempty"` + + // recall metadata + Recalled bool `json:"recalled"` + RecallHistno int `json:"recallHistno,omitempty"` + RecallStrategy string `json:"recallStrategy,omitempty"` + RecallActionsRaw string `json:"recallActionsRaw,omitempty"` + RecallActions []string `json:"recallActions,omitempty"` + RecallLastCmdLine string `json:"recallLastCmdLine"` + + // recall command + RecallPrefix string `json:"recallPrefix,omitempty"` + + // added by sanitizatizer + Sanitized bool `json:"sanitized,omitempty"` + CmdLength int `json:"cmdLength,omitempty"` + + // fields that are string here and int in older resh verisons + Cols interface{} `json:"cols"` + Lines interface{} `json:"lines"` +} diff --git a/internal/record/record.go b/internal/record/record.go new file mode 100644 index 0000000..07ad798 --- /dev/null +++ b/internal/record/record.go @@ -0,0 +1,2 @@ +// Package record provides record types that are used in resh history files +package record diff --git a/internal/record/v1.go b/internal/record/v1.go new file mode 100644 index 0000000..606988a --- /dev/null +++ b/internal/record/v1.go @@ -0,0 +1,58 @@ +package record + +type V1 struct { + // flags + // deleted, favorite + // FIXME: is this the best way? .. what about string, separate fields, or something similar + Flags int `json:"flags"` + + DeviceID string `json:"deviceID"` + SessionID string `json:"sessionID"` + // can we have a shorter uuid for record + RecordID string `json:"recordID"` + + // cmdline, exitcode + CmdLine string `json:"cmdLine"` + ExitCode int `json:"exitCode"` + + // paths + Home string `json:"home"` + Pwd string `json:"pwd"` + RealPwd string `json:"realPwd"` + + // hostname + lognem (not sure if we actually need logname) + Logname string `json:"logname"` + Hostname string `json:"hostname"` + + // git info + // origin is the most important + GitOriginRemote string `json:"gitOriginRemote"` + // maybe branch could be useful - e.g. in monorepo ?? + GitBranch string `json:"gitBranch"` + + // what is this for ?? + // session watching needs this + // but I'm not sure if we need to save it + // records belong to sessions + // PID int `json:"pid"` + // needed for tracking of sessions but I think it shouldn't be part of V1 + SessionPID int `json:"sessionPID"` + + // needed to because records are merged with parts with same "SessionID + Shlvl" + // I don't think we need to save it + Shlvl int `json:"shlvl"` + + // time (before), duration of command + Time float64 `json:"time"` + Duration float64 `json:"duration"` + + // these look like internal stuff + + // records come in two parts (collect and postcollect) + PartOne bool `json:"partOne,omitempty"` // false => part two + PartsNotMerged bool `json:"partsNotMerged,omitempty"` + + // special flag -> not an actual record but an session end + // TODO: this shouldn't be part of serializable V1 record + SessionExit bool `json:"sessionExit,omitempty"` +} diff --git a/internal/recordint/collect.go b/internal/recordint/collect.go new file mode 100644 index 0000000..b1e62e7 --- /dev/null +++ b/internal/recordint/collect.go @@ -0,0 +1,21 @@ +package recordint + +import "github.com/curusarn/resh/internal/record" + +type Collect struct { + // record merging + SessionID string + Shlvl int + // session watching + SessionPID int + + Rec record.V1 +} + +type Postcollect struct { + // record merging + SessionID string + Shlvl int + // session watching + SessionPID int +} diff --git a/internal/recordint/enriched.go b/internal/recordint/enriched.go new file mode 100644 index 0000000..b2141a4 --- /dev/null +++ b/internal/recordint/enriched.go @@ -0,0 +1,51 @@ +package recordint + +import ( + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recutil" +) + +// TODO: This all seems excessive +// TODO: V1 should be converted directly to SearchApp record + +// EnrichedRecord - record enriched with additional data +type Enriched struct { + // TODO: think about if it really makes sense to have this based on V1 + record.V1 + + // TODO: drop some/all of this + // enriching fields - added "later" + Command string `json:"command"` + FirstWord string `json:"firstWord"` + Invalid bool `json:"invalid"` + SeqSessionID uint64 `json:"seqSessionId"` + LastRecordOfSession bool `json:"lastRecordOfSession"` + DebugThisRecord bool `json:"debugThisRecord"` + Errors []string `json:"errors"` + // SeqSessionID uint64 `json:"seqSessionId,omitempty"` +} + +// Enriched - returns enriched record +func NewEnrichedFromV1(r *record.V1) Enriched { + rec := Enriched{Record: r} + // normlize git remote + rec.GitOriginRemote = NormalizeGitRemote(rec.GitOriginRemote) + rec.GitOriginRemoteAfter = NormalizeGitRemote(rec.GitOriginRemoteAfter) + // Get command/first word from commandline + var err error + err = recutil.Validate(r) + if err != nil { + rec.Errors = append(rec.Errors, "Validate error:"+err.Error()) + // rec, _ := record.ToString() + // sugar.Println("Invalid command:", rec) + rec.Invalid = true + } + rec.Command, rec.FirstWord, err = GetCommandAndFirstWord(r.CmdLine) + if err != nil { + rec.Errors = append(rec.Errors, "GetCommandAndFirstWord error:"+err.Error()) + // rec, _ := record.ToString() + // sugar.Println("Invalid command:", rec) + rec.Invalid = true // should this be really invalid ? + } + return rec +} diff --git a/internal/recordint/flag.go b/internal/recordint/flag.go new file mode 100644 index 0000000..2eaff6a --- /dev/null +++ b/internal/recordint/flag.go @@ -0,0 +1,9 @@ +package recordint + +type Flag struct { + deviceID string + recordID string + + flagDeleted bool + flagFavourite bool +} diff --git a/internal/recordint/indexed.go b/internal/recordint/indexed.go new file mode 100644 index 0000000..6d21870 --- /dev/null +++ b/internal/recordint/indexed.go @@ -0,0 +1,9 @@ +package recordint + +import "github.com/curusarn/resh/internal/record" + +// Indexed record allows us to find records in history file in order to edit them +type Indexed struct { + Rec record.V1 + Idx int +} diff --git a/internal/recordint/recordint.go b/internal/recordint/recordint.go new file mode 100644 index 0000000..73457cd --- /dev/null +++ b/internal/recordint/recordint.go @@ -0,0 +1,2 @@ +// Package recordint provides internal record types that are passed between resh components +package recordint diff --git a/internal/recordint/searchapp.go b/internal/recordint/searchapp.go new file mode 100644 index 0000000..965ebce --- /dev/null +++ b/internal/recordint/searchapp.go @@ -0,0 +1,40 @@ +package recordint + +// SearchApp record used for sending records to RESH-CLI +type SearchApp struct { + IsRaw bool + SessionID string + DeviceID string + + CmdLine string + Host string + Pwd string + Home string // helps us to collapse /home/user to tilde + GitOriginRemote string + ExitCode int + + Time float64 +} + +// NewCliRecordFromCmdLine +func NewSearchAppFromCmdLine(cmdLine string) SearchApp { + return SearchApp{ + IsRaw: true, + CmdLine: cmdLine, + } +} + +// NewCliRecord from EnrichedRecord +func NewSearchApp(r *Enriched) SearchApp { + return SearchApp{ + IsRaw: false, + SessionID: r.SessionID, + CmdLine: r.CmdLine, + Host: r.Hostname, + Pwd: r.Pwd, + Home: r.Home, + GitOriginRemote: r.GitOriginRemote, + ExitCode: r.ExitCode, + Time: r.Time, + } +} diff --git a/internal/records/records.go b/internal/records/records.go index b1f9a85..1747547 100644 --- a/internal/records/records.go +++ b/internal/records/records.go @@ -3,16 +3,13 @@ package records import ( "bufio" "encoding/json" - "errors" "fmt" "io" - "math" "os" "strconv" "strings" "github.com/curusarn/resh/internal/histlist" - "github.com/mattn/go-shellwords" "go.uber.org/zap" ) @@ -134,61 +131,6 @@ type FallbackRecord struct { Lines int `json:"lines"` // notice the int type } -// SlimRecord used for recalling because unmarshalling record w/ 50+ fields is too slow -type SlimRecord struct { - SessionID string `json:"sessionId"` - RecallHistno int `json:"recallHistno,omitempty"` - RecallPrefix string `json:"recallPrefix,omitempty"` - - // extra recall - we might use these in the future - // Pwd string `json:"pwd"` - // RealPwd string `json:"realPwd"` - // GitDir string `json:"gitDir"` - // GitRealDir string `json:"gitRealDir"` - // GitOriginRemote string `json:"gitOriginRemote"` - -} - -// CliRecord used for sending records to RESH-CLI -type CliRecord struct { - IsRaw bool `json:"isRaw"` - SessionID string `json:"sessionId"` - - CmdLine string `json:"cmdLine"` - Host string `json:"host"` - Pwd string `json:"pwd"` - Home string `json:"home"` // helps us to collapse /home/user to tilde - GitOriginRemote string `json:"gitOriginRemote"` - ExitCode int `json:"exitCode"` - - RealtimeBefore float64 `json:"realtimeBefore"` - // RealtimeAfter float64 `json:"realtimeAfter"` - // RealtimeDuration float64 `json:"realtimeDuration"` -} - -// NewCliRecordFromCmdLine from EnrichedRecord -func NewCliRecordFromCmdLine(cmdLine string) CliRecord { - return CliRecord{ - IsRaw: true, - CmdLine: cmdLine, - } -} - -// NewCliRecord from EnrichedRecord -func NewCliRecord(r EnrichedRecord) CliRecord { - return CliRecord{ - IsRaw: false, - SessionID: r.SessionID, - CmdLine: r.CmdLine, - Host: r.Host, - Pwd: r.Pwd, - Home: r.Home, - GitOriginRemote: r.GitOriginRemote, - ExitCode: r.ExitCode, - RealtimeBefore: r.RealtimeBefore, - } -} - // Convert from FallbackRecord to Record func Convert(r *FallbackRecord) Record { return Record{ @@ -208,308 +150,6 @@ func (r EnrichedRecord) ToString() (string, error) { return string(jsonRec), nil } -// Enriched - returnd enriched record -func Enriched(r Record) EnrichedRecord { - record := EnrichedRecord{Record: r} - // normlize git remote - record.GitOriginRemote = NormalizeGitRemote(record.GitOriginRemote) - record.GitOriginRemoteAfter = NormalizeGitRemote(record.GitOriginRemoteAfter) - // Get command/first word from commandline - var err error - err = r.Validate() - if err != nil { - record.Errors = append(record.Errors, "Validate error:"+err.Error()) - // rec, _ := record.ToString() - // sugar.Println("Invalid command:", rec) - record.Invalid = true - } - record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine) - if err != nil { - record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error()) - // rec, _ := record.ToString() - // sugar.Println("Invalid command:", rec) - record.Invalid = true // should this be really invalid ? - } - return record -} - -// Merge two records (part1 - collect + part2 - postcollect) -func (r *Record) Merge(r2 Record) error { - if r.PartOne == false || r2.PartOne { - return errors.New("Expected part1 and part2 of the same record - usage: part1.Merge(part2)") - } - if r.SessionID != r2.SessionID { - return errors.New("Records to merge are not from the same sesion - r1:" + r.SessionID + " r2:" + r2.SessionID) - } - if r.CmdLine != r2.CmdLine { - return errors.New("Records to merge are not parts of the same records - r1:" + r.CmdLine + " r2:" + r2.CmdLine) - } - if r.RecordID != r2.RecordID { - return errors.New("Records to merge do not have the same ID - r1:" + r.RecordID + " r2:" + r2.RecordID) - } - // r.RealtimeBefore != r2.RealtimeBefore - can't be used because of bash-preexec runs when it's not supposed to - r.ExitCode = r2.ExitCode - r.PwdAfter = r2.PwdAfter - r.RealPwdAfter = r2.RealPwdAfter - r.GitDirAfter = r2.GitDirAfter - r.GitRealDirAfter = r2.GitRealDirAfter - r.RealtimeAfter = r2.RealtimeAfter - r.GitOriginRemoteAfter = r2.GitOriginRemoteAfter - r.TimezoneAfter = r2.TimezoneAfter - r.RealtimeAfterLocal = r2.RealtimeAfterLocal - r.RealtimeDuration = r2.RealtimeDuration - - r.PartsMerged = true - r.PartOne = false - return nil -} - -// Validate - returns error if the record is invalid -func (r *Record) Validate() error { - if r.CmdLine == "" { - return errors.New("There is no CmdLine") - } - if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 { - return errors.New("There is no Time") - } - if r.RealtimeBeforeLocal == 0 || r.RealtimeAfterLocal == 0 { - return errors.New("There is no Local Time") - } - if r.RealPwd == "" || r.RealPwdAfter == "" { - return errors.New("There is no Real Pwd") - } - if r.Pwd == "" || r.PwdAfter == "" { - return errors.New("There is no Pwd") - } - - // TimezoneBefore - // TimezoneAfter - - // RealtimeDuration - // RealtimeSinceSessionStart - TODO: add later - // RealtimeSinceBoot - TODO: add later - - // device extras - // Host - // Hosttype - // Ostype - // Machtype - // OsReleaseID - // OsReleaseVersionID - // OsReleaseIDLike - // OsReleaseName - // OsReleasePrettyName - - // session extras - // Term - // Shlvl - - // static info - // Lang - // LcAll - - // meta - // ReshUUID - // ReshVersion - // ReshRevision - - // added by sanitizatizer - // Sanitized - // CmdLength - return nil -} - -// SetCmdLine sets cmdLine and related members -func (r *EnrichedRecord) SetCmdLine(cmdLine string) { - r.CmdLine = cmdLine - r.CmdLength = len(cmdLine) - r.ExitCode = 0 - var err error - r.Command, r.FirstWord, err = GetCommandAndFirstWord(cmdLine) - if err != nil { - r.Errors = append(r.Errors, "GetCommandAndFirstWord error:"+err.Error()) - // sugar.Println("Invalid command:", r.CmdLine) - r.Invalid = true - } -} - -// Stripped returns record stripped of all info that is not available during prediction -func Stripped(r EnrichedRecord) EnrichedRecord { - // clear the cmd itself - r.SetCmdLine("") - // replace after info with before info - r.PwdAfter = r.Pwd - r.RealPwdAfter = r.RealPwd - r.TimezoneAfter = r.TimezoneBefore - r.RealtimeAfter = r.RealtimeBefore - r.RealtimeAfterLocal = r.RealtimeBeforeLocal - // clear some more stuff - r.RealtimeDuration = 0 - r.LastRecordOfSession = false - return r -} - -// GetCommandAndFirstWord func -func GetCommandAndFirstWord(cmdLine string) (string, string, error) { - args, err := shellwords.Parse(cmdLine) - if err != nil { - // Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") - return "", "", err - } - if len(args) == 0 { - return "", "", nil - } - i := 0 - for true { - // commands in shell sometimes look like this `variable=something command argument otherArgument --option` - // to get the command we skip over tokens that contain '=' - if strings.ContainsRune(args[i], '=') && len(args) > i+1 { - i++ - continue - } - return args[i], args[0], nil - } - return "ERROR", "ERROR", errors.New("failed to retrieve first word of command") -} - -// NormalizeGitRemote func -func NormalizeGitRemote(gitRemote string) string { - if strings.HasSuffix(gitRemote, ".git") { - return gitRemote[:len(gitRemote)-4] - } - return gitRemote -} - -// DistParams is used to supply params to Enrichedrecords.DistanceTo() -type DistParams struct { - ExitCode float64 - MachineID float64 - SessionID float64 - Login float64 - Shell float64 - Pwd float64 - RealPwd float64 - Git float64 - Time float64 -} - -// DistanceTo another record -func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { - var dist float64 - dist = 0 - - // lev distance or something? TODO later - // CmdLine - - // exit code - if r.ExitCode != r2.ExitCode { - if r.ExitCode == 0 || r2.ExitCode == 0 { - // one success + one error -> 1 - dist += 1 * p.ExitCode - } else { - // two different errors - dist += 0.5 * p.ExitCode - } - } - - // machine/device - if r.MachineID != r2.MachineID { - dist += 1 * p.MachineID - } - // Uname - - // session - if r.SessionID != r2.SessionID { - dist += 1 * p.SessionID - } - // Pid - add because of nested shells? - // SessionPid - - // user - if r.Login != r2.Login { - dist += 1 * p.Login - } - // Home - - // shell - if r.Shell != r2.Shell { - dist += 1 * p.Shell - } - // ShellEnv - - // pwd - if r.Pwd != r2.Pwd { - // TODO: compare using hierarchy - // TODO: make more important - dist += 1 * p.Pwd - } - if r.RealPwd != r2.RealPwd { - // TODO: -||- - dist += 1 * p.RealPwd - } - // PwdAfter - // RealPwdAfter - - // git - if r.GitDir != r2.GitDir { - dist += 1 * p.Git - } - if r.GitRealDir != r2.GitRealDir { - dist += 1 * p.Git - } - if r.GitOriginRemote != r2.GitOriginRemote { - dist += 1 * p.Git - } - - // time - // this can actually get negative for differences of less than one second which is fine - // distance grows by 1 with every order - distTime := math.Log10(math.Abs(r.RealtimeBefore-r2.RealtimeBefore)) * p.Time - if math.IsNaN(distTime) == false && math.IsInf(distTime, 0) == false { - dist += distTime - } - // RealtimeBeforeLocal - // RealtimeAfter - // RealtimeAfterLocal - - // TimezoneBefore - // TimezoneAfter - - // RealtimeDuration - // RealtimeSinceSessionStart - TODO: add later - // RealtimeSinceBoot - TODO: add later - - // device extras - // Host - // Hosttype - // Ostype - // Machtype - // OsReleaseID - // OsReleaseVersionID - // OsReleaseIDLike - // OsReleaseName - // OsReleasePrettyName - - // session extras - // Term - // Shlvl - - // static info - // Lang - // LcAll - - // meta - // ReshUUID - // ReshVersion - // ReshRevision - - // added by sanitizatizer - // Sanitized - // CmdLength - - return dist -} - // LoadFromFile loads records from 'fname' file func LoadFromFile(sugar *zap.SugaredLogger, fname string) []Record { const allowedErrors = 3 diff --git a/internal/records/records_test.go b/internal/records/records_test.go index 35c6920..9bcb69f 100644 --- a/internal/records/records_test.go +++ b/internal/records/records_test.go @@ -77,75 +77,9 @@ func TestValidate(t *testing.T) { } } -func TestSetCmdLine(t *testing.T) { - record := EnrichedRecord{} - cmdline := "cmd arg1 arg2" - record.SetCmdLine(cmdline) - if record.CmdLine != cmdline || record.Command != "cmd" || record.FirstWord != "cmd" { - t.Error() - } -} - -func TestStripped(t *testing.T) { - for _, rec := range GetTestEnrichedRecords() { - stripped := Stripped(rec) - - // there should be no cmdline - if stripped.CmdLine != "" || - stripped.FirstWord != "" || - stripped.Command != "" { - t.Error("Stripped() returned record w/ info about CmdLine, Command OR FirstWord") - } - // *after* fields should be overwritten by *before* fields - if stripped.PwdAfter != stripped.Pwd || - stripped.RealPwdAfter != stripped.RealPwd || - stripped.TimezoneAfter != stripped.TimezoneBefore || - stripped.RealtimeAfter != stripped.RealtimeBefore || - stripped.RealtimeAfterLocal != stripped.RealtimeBeforeLocal { - t.Error("Stripped() returned record w/ different *after* and *before* values - *after* fields should be overwritten by *before* fields") - } - // there should be no information about duration and session end - if stripped.RealtimeDuration != 0 || - stripped.LastRecordOfSession != false { - t.Error("Stripped() returned record with too much information") - } - } -} - func TestGetCommandAndFirstWord(t *testing.T) { cmd, stWord, err := GetCommandAndFirstWord("cmd arg1 arg2") if err != nil || cmd != "cmd" || stWord != "cmd" { t.Error("GetCommandAndFirstWord() returned wrong Command OR FirstWord") } } - -func TestDistanceTo(t *testing.T) { - paramsFull := DistParams{ - ExitCode: 1, - MachineID: 1, - SessionID: 1, - Login: 1, - Shell: 1, - Pwd: 1, - RealPwd: 1, - Git: 1, - Time: 1, - } - paramsZero := DistParams{} - var prevRec EnrichedRecord - for _, rec := range GetTestEnrichedRecords() { - dist := rec.DistanceTo(rec, paramsFull) - if dist != 0 { - t.Error("DistanceTo() itself should be always 0") - } - dist = rec.DistanceTo(prevRec, paramsFull) - if dist == 0 { - t.Error("DistanceTo() between two test records shouldn't be 0") - } - dist = rec.DistanceTo(prevRec, paramsZero) - if dist != 0 { - t.Error("DistanceTo() should be 0 when DistParams is all zeros") - } - prevRec = rec - } -} diff --git a/internal/recutil/recutil.go b/internal/recutil/recutil.go new file mode 100644 index 0000000..96d8baa --- /dev/null +++ b/internal/recutil/recutil.go @@ -0,0 +1,94 @@ +package recutil + +import ( + "errors" + "net/url" + "strings" + + "github.com/curusarn/resh/internal/record" + "github.com/mattn/go-shellwords" + giturls "github.com/whilp/git-urls" +) + +// NormalizeGitRemote helper +func NormalizeGitRemote(gitRemote string) string { + if strings.HasSuffix(gitRemote, ".git") { + gitRemote = gitRemote[:len(gitRemote)-4] + } + parsedURL, err := giturls.Parse(gitRemote) + if err != nil { + // TODO: log this error + return gitRemote + } + if parsedURL.User == nil || parsedURL.User.Username() == "" { + parsedURL.User = url.User("git") + } + // TODO: figure out what scheme we want + parsedURL.Scheme = "git+ssh" + return parsedURL.String() +} + +// Validate returns error if the record is invalid +func Validate(r *record.V1) error { + if r.CmdLine == "" { + return errors.New("There is no CmdLine") + } + if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 { + return errors.New("There is no Time") + } + if r.RealtimeBeforeLocal == 0 || r.RealtimeAfterLocal == 0 { + return errors.New("There is no Local Time") + } + if r.RealPwd == "" || r.RealPwdAfter == "" { + return errors.New("There is no Real Pwd") + } + if r.Pwd == "" || r.PwdAfter == "" { + return errors.New("There is no Pwd") + } + return nil +} + +// Merge two records (part1 - collect + part2 - postcollect) +func Merge(r1 *record.V1, r2 *record.V1) error { + if r1.PartOne == false || r2.PartOne { + return errors.New("Expected part1 and part2 of the same record - usage: Merge(part1, part2)") + } + if r1.SessionID != r2.SessionID { + return errors.New("Records to merge are not from the same sesion - r1:" + r1.SessionID + " r2:" + r2.SessionID) + } + if r1.CmdLine != r2.CmdLine { + return errors.New("Records to merge are not parts of the same records - r1:" + r1.CmdLine + " r2:" + r2.CmdLine) + } + if r1.RecordID != r2.RecordID { + return errors.New("Records to merge do not have the same ID - r1:" + r1.RecordID + " r2:" + r2.RecordID) + } + r1.ExitCode = r2.ExitCode + r1.Duration = r2.Duration + + r1.PartsMerged = true + r1.PartOne = false + return nil +} + +// GetCommandAndFirstWord func +func GetCommandAndFirstWord(cmdLine string) (string, string, error) { + args, err := shellwords.Parse(cmdLine) + if err != nil { + // Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") + return "", "", err + } + if len(args) == 0 { + return "", "", nil + } + i := 0 + for true { + // commands in shell sometimes look like this `variable=something command argument otherArgument --option` + // to get the command we skip over tokens that contain '=' + if strings.ContainsRune(args[i], '=') && len(args) > i+1 { + i++ + continue + } + return args[i], args[0], nil + } + return "ERROR", "ERROR", errors.New("failed to retrieve first word of command") +} diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 5d9ebe3..a734b7b 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -29,19 +29,7 @@ __resh_collect() { local __RESH_GIT_REMOTE; __RESH_GIT_REMOTE="$(git remote get-url origin 2>/dev/null)" local __RESH_GIT_REMOTE_EXIT_CODE=$? - if [ -n "${ZSH_VERSION-}" ]; then - # assume Zsh - local __RESH_PID="$$" # current pid - elif [ -n "${BASH_VERSION-}" ]; then - # assume Bash - if [ "${BASH_VERSINFO[0]}" -ge "4" ]; then - # $BASHPID is only available in bash4+ - # $$ is fairly similar so it should not be an issue - local __RESH_PID="$BASHPID" # current pid - else - local __RESH_PID="$$" # current pid - fi - fi + local __RESH_PID="$$" # current pid # time local __RESH_TZ_BEFORE; __RESH_TZ_BEFORE=$(date +%z) # __RESH_RT_BEFORE="$EPOCHREALTIME" diff --git a/scripts/install.sh b/scripts/install.sh index b07ac4d..8d5aba9 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -94,6 +94,19 @@ fi # read -r x echo +echo "Backing up previous installation" +# TODO: ~/.resh -> XDG_DATA/resh/rollback/ +# TODO: ~/XDG_DATA/resh/history.reshjson -> XDG_DATA/resh/rollback/ +# TODO: what about legacy history locations +# TODO: ~/XDG_DATA/resh/log.json -> XDG_DATA/resh/rollback/ + +echo "Cleaning up installation directory ..." +rm ~/.resh/bin/* 2>/dev/null ||: +rm ~/.resh/* 2>/dev/null 2>/dev/null ||: +# TODO: put this behind version condition +# backward compatibility: We have a new location for resh history file +[ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json + echo "Creating directories ..." mkdir_if_not_exists() { @@ -122,15 +135,9 @@ bin/resh-control completion zsh > ~/.resh/zsh_completion.d/_reshctl echo "Copying more files ..." cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid -rm ~/.resh/bin/resh-* ||: cp -f bin/resh-{daemon,control,collect,postcollect,session-init,config} ~/.resh/bin/ -cp -f scripts/resh-evaluate-plot.py ~/.resh/bin/ -cp -fr data/sanitizer ~/.resh/sanitizer_data - -# backward compatibility: We have a new location for resh history file -[ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json -echo "Checking config file ..." +echo "Creating/updating config file ..." ./bin/resh-config-setup echo "Finishing up ..." diff --git a/scripts/resh-evaluate-plot.py b/scripts/resh-evaluate-plot.py deleted file mode 100755 index 89792cb..0000000 --- a/scripts/resh-evaluate-plot.py +++ /dev/null @@ -1,1218 +0,0 @@ -#!/usr/bin/env python3 - - -import traceback -import sys -import json -from collections import defaultdict -import numpy as np -from graphviz import Digraph -from datetime import datetime - -from matplotlib import rcParams -rcParams['font.family'] = 'serif' -# rcParams['font.serif'] = [''] - -import matplotlib.pyplot as plt -import matplotlib.path as mpath -import matplotlib.patches as mpatches - -PLOT_WIDTH = 10 # inches -PLOT_HEIGHT = 7 # inches - -PLOT_SIZE_zipf = 20 - -data = json.load(sys.stdin) - -DATA_records = [] -DATA_records_by_session = defaultdict(list) -DATA_records_by_user = defaultdict(list) -for user in data["UsersRecords"]: - if user["Devices"] is None: - continue - for device in user["Devices"]: - if device["Records"] is None: - continue - for record in device["Records"]: - if "invalid" in record and record["invalid"]: - continue - - DATA_records.append(record) - DATA_records_by_session[record["seqSessionId"]].append(record) - DATA_records_by_user[user["Name"] + ":" + device["Name"]].append(record) - -DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"])) - -for pid, session in DATA_records_by_session.items(): - session = list(sorted(session, key=lambda x: x["realtimeAfterLocal"])) - -# TODO: this should be a cmdline option -async_draw = True - -# for strategy in data["Strategies"]: -# print(json.dumps(strategy)) - -def zipf(length): - return list(map(lambda x: 1/2**x, range(0, length))) - - -def trim(text, length, add_elipse=True): - if add_elipse and len(text) > length: - return text[:length-1] + "…" - return text[:length] - - -# Figure 3.1. The normalized command frequency, compared with Zipf. -def plot_cmdLineFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): - cmdLine_count = defaultdict(int) - for record in DATA_records: - cmdLine_count[record["cmdLine"]] += 1 - - tmp = sorted(cmdLine_count.items(), key=lambda x: x[1], reverse=True)[:plotSize] - cmdLineFrq = list(map(lambda x: x[1] / tmp[0][1], tmp)) - labels = list(map(lambda x: trim(x[0], 7), tmp)) - - ranks = range(1, len(cmdLineFrq)+1) - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.plot(ranks, zipf(len(ranks)), '-') - plt.plot(ranks, cmdLineFrq, 'o-') - plt.title("Commandline frequency / rank") - plt.ylabel("Normalized commandline frequency") - plt.xlabel("Commandline rank") - plt.legend(("Zipf", "Commandline"), loc="best") - if show_labels: - plt.xticks(ranks, labels, rotation=-60) - # TODO: make xticks integral - if async_draw: - plt.draw() - else: - plt.show() - - -# similar to ~ Figure 3.1. The normalized command frequency, compared with Zipf. -def plot_cmdFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command frequency / rank") - plt.ylabel("Normalized command frequency") - plt.xlabel("Command rank") - legend = [] - - - cmd_count = defaultdict(int) - len_records = 0 - for record in DATA_records: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - len_records += 1 - - tmp = sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)[:plotSize] - cmdFrq = list(map(lambda x: x[1] / tmp[0][1], tmp)) - labels = list(map(lambda x: trim(x[0], 7), tmp)) - - top100percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(1 * len(cmd_count))])) / len_records - top10percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.1 * len(cmd_count))])) / len_records - top20percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.2 * len(cmd_count))])) / len_records - print("% ALL: Top {} %% of cmds amounts for {} %% of all command lines".format(100, top100percent)) - print("% ALL: Top {} %% of cmds amounts for {} %% of all command lines".format(10, top10percent)) - print("% ALL: Top {} %% of cmds amounts for {} %% of all command lines".format(20, top20percent)) - ranks = range(1, len(cmdFrq)+1) - plt.plot(ranks, zipf(len(ranks)), '-') - legend.append("Zipf distribution") - plt.plot(ranks, cmdFrq, 'o-') - legend.append("All subjects") - - - for user in DATA_records_by_user.items(): - cmd_count = defaultdict(int) - len_records = 0 - name, records = user - for record in records: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - len_records += 1 - - tmp = sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)[:plotSize] - cmdFrq = list(map(lambda x: x[1] / tmp[0][1], tmp)) - labels = list(map(lambda x: trim(x[0], 7), tmp)) - - top100percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(1 * len(cmd_count))])) / len_records - top10percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.1 * len(cmd_count))])) / len_records - top20percent = 100 * sum(map(lambda x: x[1], list(cmd_count.items())[:int(0.2 * len(cmd_count))])) / len_records - print("% {}: Top {} %% of cmds amounts for {} %% of all command lines".format(name, 100, top100percent)) - print("% {}: Top {} %% of cmds amounts for {} %% of all command lines".format(name, 10, top10percent)) - print("% {}: Top {} %% of cmds amounts for {} %% of all command lines".format(name, 20, top20percent)) - ranks = range(1, len(cmdFrq)+1) - plt.plot(ranks, cmdFrq, 'o-') - legend.append("{} (sanitize!)".format(name)) - - plt.legend(legend, loc="best") - - if show_labels: - plt.xticks(ranks, labels, rotation=-60) - # TODO: make xticks integral - if async_draw: - plt.draw() - else: - plt.show() - -# Figure 3.2. Command vocabulary size vs. the number of command lines entered for four individuals. -def plot_cmdVocabularySize_cmdLinesEntered(): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command vocabulary size vs. the number of command lines entered") - plt.ylabel("Command vocabulary size") - plt.xlabel("# of command lines entered") - legend = [] - - # x_count = max(map(lambda x: len(x[1]), DATA_records_by_user.items())) - # x_values = range(0, x_count) - for user in DATA_records_by_user.items(): - new_cmds_after_1k = 0 - new_cmds_after_2k = 0 - new_cmds_after_3k = 0 - cmd_vocabulary = set() - y_cmd_count = [0] - name, records = user - for record in records: - cmd = record["command"] - if cmd == "": - continue - if cmd in cmd_vocabulary: - # repeat last value - y_cmd_count.append(y_cmd_count[-1]) - else: - cmd_vocabulary.add(cmd) - # append last value +1 - y_cmd_count.append(y_cmd_count[-1] + 1) - if len(y_cmd_count) > 1000: - new_cmds_after_1k+=1 - if len(y_cmd_count) > 2000: - new_cmds_after_2k+=1 - if len(y_cmd_count) > 3000: - new_cmds_after_3k+=1 - - if len(y_cmd_count) == 1000: - print("% {}: Cmd adoption rate at 1k (between 0 and 1k) cmdlines = {}".format(name ,len(cmd_vocabulary) / (len(y_cmd_count)))) - if len(y_cmd_count) == 2000: - print("% {}: Cmd adoption rate at 2k cmdlines = {}".format(name ,len(cmd_vocabulary) / (len(y_cmd_count)))) - print("% {}: Cmd adoption rate between 1k and 2k cmdlines = {}".format(name ,new_cmds_after_1k / (len(y_cmd_count) - 1000))) - if len(y_cmd_count) == 3000: - print("% {}: Cmd adoption rate between 2k and 3k cmdlines = {}".format(name ,new_cmds_after_2k / (len(y_cmd_count) - 2000))) - - print("% {}: New cmd adoption rate after 1k cmdlines = {}".format(name ,new_cmds_after_1k / (len(y_cmd_count) - 1000))) - print("% {}: New cmd adoption rate after 2k cmdlines = {}".format(name ,new_cmds_after_2k / (len(y_cmd_count) - 2000))) - print("% {}: New cmd adoption rate after 3k cmdlines = {}".format(name ,new_cmds_after_3k / (len(y_cmd_count) - 3000))) - x_cmds_entered = range(0, len(y_cmd_count)) - plt.plot(x_cmds_entered, y_cmd_count, '-') - legend.append(name + " (TODO: sanitize!)") - - # print(cmd_vocabulary) - - plt.legend(legend, loc="best") - - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_cmdVocabularySize_daily(): - SECONDS_IN_A_DAY = 86400 - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command vocabulary size in days") - plt.ylabel("Command vocabulary size") - plt.xlabel("Days") - legend = [] - - # x_count = max(map(lambda x: len(x[1]), DATA_records_by_user.items())) - # x_values = range(0, x_count) - for user in DATA_records_by_user.items(): - new_cmds_after_100 = 0 - new_cmds_after_200 = 0 - new_cmds_after_300 = 0 - cmd_vocabulary = set() - y_cmd_count = [0] - name, records = user - - cmd_fail_count = 0 - - if not len(records): - print("ERROR: no records for user {}".format(name)) - continue - - first_day = records[0]["realtimeAfter"] - this_day = first_day - - for record in records: - cmd = record["command"] - timestamp = record["realtimeAfter"] - - if cmd == "": - cmd_fail_count += 1 - continue - - if timestamp >= this_day + SECONDS_IN_A_DAY: - this_day += SECONDS_IN_A_DAY - while timestamp >= this_day + SECONDS_IN_A_DAY: - y_cmd_count.append(-10) - this_day += SECONDS_IN_A_DAY - - y_cmd_count.append(len(cmd_vocabulary)) - cmd_vocabulary = set() # wipes the vocabulary each day - - if len(y_cmd_count) > 100: - new_cmds_after_100+=1 - if len(y_cmd_count) > 200: - new_cmds_after_200+=1 - if len(y_cmd_count) > 300: - new_cmds_after_300+=1 - - if len(y_cmd_count) == 100: - print("% {}: Cmd adoption rate at 100 days (between 0 and 100 days) = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - if len(y_cmd_count) == 200: - print("% {}: Cmd adoption rate at 200 days days = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - print("% {}: Cmd adoption rate between 100 and 200 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - if len(y_cmd_count) == 300: - print("% {}: Cmd adoption rate between 200 and 300 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - - if cmd not in cmd_vocabulary: - cmd_vocabulary.add(cmd) - - - print("% {}: New cmd adoption rate after 100 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - print("% {}: New cmd adoption rate after 200 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - print("% {}: New cmd adoption rate after 300 days = {}".format(name, new_cmds_after_300 / (len(y_cmd_count) - 300))) - print("% {}: cmd_fail_count = {}".format(name, cmd_fail_count)) - x_cmds_entered = range(0, len(y_cmd_count)) - plt.plot(x_cmds_entered, y_cmd_count, 'o', markersize=2) - legend.append(name + " (TODO: sanitize!)") - - # print(cmd_vocabulary) - - plt.legend(legend, loc="best") - plt.ylim(bottom=-5) - - if async_draw: - plt.draw() - else: - plt.show() - - -def matplotlib_escape(ss): - ss = ss.replace('$', '\\$') - return ss - - -def plot_cmdUsage_in_time(sort_cmds=False, num_cmds=None): - SECONDS_IN_A_DAY = 86400 - tab_colors = ("tab:blue", "tab:orange", "tab:green", "tab:red", "tab:purple", "tab:brown", "tab:pink", "tab:gray") - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command use in time") - plt.ylabel("Commands") - plt.xlabel("Days") - legend_patches = [] - - cmd_ids = {} - y_labels = [] - - all_x_values = [] - all_y_values = [] - all_s_values = [] # size - all_c_values = [] # color - - x_values = [] - y_values = [] - s_values = [] # size - c_values = [] # color - - if sort_cmds: - cmd_count = defaultdict(int) - for user in DATA_records_by_user.items(): - name, records = user - for record in records: - cmd = record["command"] - cmd_count[cmd] += 1 - - sorted_cmds = map(lambda x: x[0], sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)) - - for cmd in sorted_cmds: - cmd_ids[cmd] = len(cmd_ids) - y_labels.append(matplotlib_escape(cmd)) - - - for user_idx, user in enumerate(DATA_records_by_user.items()): - name, records = user - - if not len(records): - print("ERROR: no records for user {}".format(name)) - continue - - - first_day = records[0]["realtimeAfter"] - this_day = first_day - day_no = 0 - today_cmds = defaultdict(int) - - for record in records: - cmd = record["command"] - timestamp = record["realtimeAfter"] - - if cmd == "": - print("NOTICE: Empty cmd for {}".format(record["cmdLine"])) - continue - - if timestamp >= this_day + SECONDS_IN_A_DAY: - for item in today_cmds.items(): - cmd, count = item - cmd_id = cmd_ids[cmd] - # skip commands with high ids - if num_cmds is not None and cmd_id >= num_cmds: - continue - - x_values.append(day_no) - y_values.append(cmd_id) - s_values.append(count) - c_values.append(tab_colors[user_idx]) - - today_cmds = defaultdict(int) - - this_day += SECONDS_IN_A_DAY - day_no += 1 - while timestamp >= this_day + SECONDS_IN_A_DAY: - this_day += SECONDS_IN_A_DAY - day_no += 1 - - if cmd not in cmd_ids: - cmd_ids[cmd] = len(cmd_ids) - y_labels.append(matplotlib_escape(cmd)) - - today_cmds[cmd] += 1 - - all_x_values.extend(x_values) - all_y_values.extend(y_values) - all_s_values.extend(s_values) - all_c_values.extend(c_values) - x_values = [] - y_values = [] - s_values = [] - c_values = [] - legend_patches.append(mpatches.Patch(color=tab_colors[user_idx], label="{} ({}) (TODO: sanitize!)".format(name, user_idx))) - - if num_cmds is not None and len(y_labels) > num_cmds: - y_labels = y_labels[:num_cmds] - plt.yticks(ticks=range(0, len(y_labels)), labels=y_labels, fontsize=6) - plt.scatter(all_x_values, all_y_values, s=all_s_values, c=all_c_values, marker='o') - plt.legend(handles=legend_patches, loc="best") - - if async_draw: - plt.draw() - else: - plt.show() - - -# Figure 5.6. Command line vocabulary size vs. the number of commands entered for four typical individuals. -def plot_cmdVocabularySize_time(): - SECONDS_IN_A_DAY = 86400 - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command vocabulary size growth in time") - plt.ylabel("Command vocabulary size") - plt.xlabel("Days") - legend = [] - - # x_count = max(map(lambda x: len(x[1]), DATA_records_by_user.items())) - # x_values = range(0, x_count) - for user in DATA_records_by_user.items(): - new_cmds_after_100 = 0 - new_cmds_after_200 = 0 - new_cmds_after_300 = 0 - cmd_vocabulary = set() - y_cmd_count = [0] - name, records = user - - cmd_fail_count = 0 - - if not len(records): - print("ERROR: no records for user {}".format(name)) - continue - - first_day = records[0]["realtimeAfter"] - this_day = first_day - - for record in records: - cmd = record["command"] - timestamp = record["realtimeAfter"] - - if cmd == "": - cmd_fail_count += 1 - continue - - if timestamp >= this_day + SECONDS_IN_A_DAY: - this_day += SECONDS_IN_A_DAY - while timestamp >= this_day + SECONDS_IN_A_DAY: - y_cmd_count.append(-10) - this_day += SECONDS_IN_A_DAY - - y_cmd_count.append(len(cmd_vocabulary)) - - if len(y_cmd_count) > 100: - new_cmds_after_100+=1 - if len(y_cmd_count) > 200: - new_cmds_after_200+=1 - if len(y_cmd_count) > 300: - new_cmds_after_300+=1 - - if len(y_cmd_count) == 100: - print("% {}: Cmd adoption rate at 100 days (between 0 and 100 days) = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - if len(y_cmd_count) == 200: - print("% {}: Cmd adoption rate at 200 days days = {}".format(name, len(cmd_vocabulary) / (len(y_cmd_count)))) - print("% {}: Cmd adoption rate between 100 and 200 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - if len(y_cmd_count) == 300: - print("% {}: Cmd adoption rate between 200 and 300 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - - if cmd not in cmd_vocabulary: - cmd_vocabulary.add(cmd) - - - print("% {}: New cmd adoption rate after 100 days = {}".format(name, new_cmds_after_100 / (len(y_cmd_count) - 100))) - print("% {}: New cmd adoption rate after 200 days = {}".format(name, new_cmds_after_200 / (len(y_cmd_count) - 200))) - print("% {}: New cmd adoption rate after 300 days = {}".format(name, new_cmds_after_300 / (len(y_cmd_count) - 300))) - print("% {}: cmd_fail_count = {}".format(name, cmd_fail_count)) - x_cmds_entered = range(0, len(y_cmd_count)) - plt.plot(x_cmds_entered, y_cmd_count, 'o', markersize=2) - legend.append(name + " (TODO: sanitize!)") - - # print(cmd_vocabulary) - - plt.legend(legend, loc="best") - plt.ylim(bottom=0) - - if async_draw: - plt.draw() - else: - plt.show() - - -# Figure 5.6. Command line vocabulary size vs. the number of commands entered for four typical individuals. -def plot_cmdLineVocabularySize_cmdLinesEntered(): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Command line vocabulary size vs. the number of command lines entered") - plt.ylabel("Command line vocabulary size") - plt.xlabel("# of command lines entered") - legend = [] - - for user in DATA_records_by_user.items(): - cmdLine_vocabulary = set() - y_cmdLine_count = [0] - name, records = user - for record in records: - cmdLine = record["cmdLine"] - if cmdLine in cmdLine_vocabulary: - # repeat last value - y_cmdLine_count.append(y_cmdLine_count[-1]) - else: - cmdLine_vocabulary.add(cmdLine) - # append last value +1 - y_cmdLine_count.append(y_cmdLine_count[-1] + 1) - - # print(cmdLine_vocabulary) - x_cmdLines_entered = range(0, len(y_cmdLine_count)) - plt.plot(x_cmdLines_entered, y_cmdLine_count, '-') - legend.append(name + " (TODO: sanitize!)") - - plt.legend(legend, loc="best") - - if async_draw: - plt.draw() - else: - plt.show() - -# Figure 3.3. Sequential structure of UNIX command usage, from Figure 4 in Hanson et al. (1984). -# Ball diameters are proportional to stationary probability. Lines indicate significant dependencies, -# solid ones being more probable (p < .0001) and dashed ones less probable (.005 < p < .0001). -def graph_cmdSequences(node_count=33, edge_minValue=0.05, view_graph=True): - START_CMD = "_start_" - END_CMD = "_end_" - cmd_count = defaultdict(int) - cmdSeq_count = defaultdict(lambda: defaultdict(int)) - cmd_id = dict() - x = 0 - cmd_id[START_CMD] = str(x) - x += 1 - cmd_id[END_CMD] = str(x) - for pid, session in DATA_records_by_session.items(): - cmd_count[START_CMD] += 1 - prev_cmd = START_CMD - for record in session: - cmd = record["command"] - if cmd == "": - continue - cmdSeq_count[prev_cmd][cmd] += 1 - cmd_count[cmd] += 1 - if cmd not in cmd_id: - x += 1 - cmd_id[cmd] = str(x) - prev_cmd = cmd - # end the session - cmdSeq_count[prev_cmd][END_CMD] += 1 - cmd_count[END_CMD] += 1 - - - # get `node_count` of largest nodes - sorted_cmd_count = sorted(cmd_count.items(), key=lambda x: x[1], reverse=True) - print(sorted_cmd_count) - cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:node_count] - - # use 3 biggest nodes as a reference point for scaling - biggest_node = cmd_count[cmds_to_graph[0]] - nd_biggest_node = cmd_count[cmds_to_graph[1]] - rd_biggest_node = cmd_count[cmds_to_graph[1]] - count2scale_coef = 3 / (biggest_node + nd_biggest_node + rd_biggest_node) - - # scaling constant - # affects node size and node label - base_scaling_factor = 21 - # extra scaling for experiments - not really useful imho - # affects everything nodes, edges, node labels, treshold for turning label into xlabel, xlabel size, ... - extra_scaling_factor = 1.0 - for x in range(0, 10): - # graphviz is not the most reliable piece of software - # -> retry on fail but scale nodes down by 1% - scaling_factor = base_scaling_factor * (1 - x * 0.01) - - # overlap: scale -> solve overlap by scaling the graph - # overlap_shrink -> try to shrink the graph a bit after you are done - # splines -> don't draw edges over nodes - # sep: 2.5 -> assume that nodes are 2.5 inches larger - graph_attr={'overlap':'scale', 'overlap_shrink':'true', - 'splines':'true', 'sep':'0.25'} - graph = Digraph(name='command_sequentiality', engine='neato', graph_attr=graph_attr) - - # iterate over all nodes - for cmd in cmds_to_graph: - seq = cmdSeq_count[cmd] - count = cmd_count[cmd] - - # iterate over all "following" commands (for each node) - for seq_entry in seq.items(): - cmd2, seq_count = seq_entry - relative_seq_count = seq_count / count - - # check if "follow" command is supposed to be in the graph - if cmd2 not in cmds_to_graph: - continue - # check if the edge value is high enough - if relative_seq_count < edge_minValue: - continue - - # create starting node and end node for the edge - # duplicates don't matter - for id_, cmd_ in ((cmd_id[cmd], cmd), (cmd_id[cmd2], cmd2)): - count_ = cmd_count[cmd_] - scale_ = count_ * count2scale_coef * scaling_factor * extra_scaling_factor - width_ = 0.08 * scale_ - fontsize_ = 8.5 * scale_ / (len(cmd_) + 3) - - width_ = str(width_) - if fontsize_ < 12 * extra_scaling_factor: - graph.node(id_, ' ', shape='circle', fixedsize='true', fontname='monospace bold', - width=width_, fontsize=str(12 * extra_scaling_factor), forcelabels='true', xlabel=cmd_) - else: - fontsize_ = str(fontsize_) - graph.node(id_, cmd_, shape='circle', fixedsize='true', fontname='monospace bold', - width=width_, fontsize=fontsize_, forcelabels='true', labelloc='c') - - # value of the edge (percentage) 1.0 is max - scale_ = seq_count / cmd_count[cmd] - penwidth_ = str((0.5 + 4.5 * scale_) * extra_scaling_factor) - #penwidth_bold_ = str(8 * scale_) - # if scale_ > 0.5: - # graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='true', splines='curved', - # penwidth=penwidth_, style='bold', arrowhead='diamond') - # elif scale_ > 0.2: - if scale_ > 0.3: - scale_ = str(int(scale_ * 100)/100) - graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='true', splines='curved', - penwidth=penwidth_, forcelables='true', label=scale_) - elif scale_ > 0.2: - graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='true', splines='curved', - penwidth=penwidth_, style='dashed') - # elif scale_ > 0.1: - else: - graph.edge(cmd_id[cmd], cmd_id[cmd2], constraint='false', splines='curved', - penwidth=penwidth_, style='dotted', arrowhead='empty') - - # graphviz sometimes fails - see above - try: - # graph.view() - graph.render('/tmp/resh-graph-command_sequence-nodeCount_{}-edgeMinVal_{}.gv'.format(node_count, edge_minValue), view=view_graph) - break - except Exception as e: - trace = traceback.format_exc() - print("GRAPHVIZ EXCEPTION: <{}>\nGRAPHVIZ TRACE: <{}>".format(str(e), trace)) - - -def plot_strategies_matches(plot_size=50, selected_strategies=[], show_strat_title=True, force_strat_title=None): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Matches at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel('%' + " of matches") - plt.xlabel("Distance") - legend = [] - x_values = range(1, plot_size+1) - saved_matches_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_matches_total = matches_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - acc = 0 - matches_cumulative = [] - for x in matches: - acc += x - matches_cumulative.append(acc) - # matches_cumulative.append(matches_total) - matches_percent = list(map(lambda x: 100 * x / dataPoint_count, matches_cumulative)) - - plt.plot(x_values, matches_percent, 'o-') - if force_strat_title is not None: - legend.append(force_strat_title) - else: - legend.append(strategy_title) - - - assert(saved_matches_total is not None) - assert(saved_dataPoint_count is not None) - max_values = [100 * saved_matches_total / saved_dataPoint_count] * len(x_values) - print("% >>> Avg recurrence rate = {}".format(max_values[0])) - plt.plot(x_values, max_values, 'r-') - legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - acc = 0 - charsRecalled_cumulative = [] - for x in charsRecalled: - acc += x - charsRecalled_cumulative.append(acc) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - print("% >>> Max avg recalled characters = {}".format(max_values[0])) - plt.plot(x_values, max_values, 'r-') - legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - plt.ylim(bottom=0) - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance (including prefix matches) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled (including prefix matches)") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for multiMatch in strategy["PrefixMatches"]: - dataPoint_count += 1 - - if not multiMatch["Match"]: - continue - matches_total += 1 - - last_charsRecalled = 0 - for match in multiMatch["Entries"]: - - chars = match["CharsRecalled"] - charsIncrease = chars - last_charsRecalled - assert(charsIncrease > 0) - charsRecalled_total += charsIncrease - - dist = match["Distance"] - if dist > plot_size: - continue - - charsRecalled[dist-1] += charsIncrease - last_charsRecalled = chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - acc = 0 - charsRecalled_cumulative = [] - for x in charsRecalled: - acc += x - charsRecalled_cumulative.append(acc) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - print("% >>> Max avg recalled characters (including prefix matches) = {}".format(max_values[0])) - plt.plot(x_values, max_values, 'r-') - legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - plt.ylim(bottom=0) - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_matches_noncummulative(plot_size=50, selected_strategies=["recent (bash-like)"], show_strat_title=False, force_strat_title=None): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Matches at distance (noncumulative) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel('%' + " of matches") - plt.xlabel("Distance") - legend = [] - x_values = range(1, plot_size+1) - saved_matches_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_matches_total = matches_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - # acc = 0 - # matches_cumulative = [] - # for x in matches: - # acc += x - # matches_cumulative.append(acc) - # # matches_cumulative.append(matches_total) - matches_percent = list(map(lambda x: 100 * x / dataPoint_count, matches)) - - plt.plot(x_values, matches_percent, 'o-') - if force_strat_title is not None: - legend.append(force_strat_title) - else: - legend.append(strategy_title) - - assert(saved_matches_total is not None) - assert(saved_dataPoint_count is not None) - # max_values = [100 * saved_matches_total / saved_dataPoint_count] * len(x_values) - # print("% >>> Avg recurrence rate = {}".format(max_values[0])) - # plt.plot(x_values, max_values, 'r-') - # legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - # plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled_noncummulative(plot_size=50, selected_strategies=["recent (bash-like)"], show_strat_title=False): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance (noncumulative) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches = [0] * plot_size - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for match in strategy["Matches"]: - dataPoint_count += 1 - - if not match["Match"]: - continue - - chars = match["CharsRecalled"] - charsRecalled_total += chars - matches_total += 1 - - dist = match["Distance"] - if dist > plot_size: - continue - - matches[dist-1] += 1 - charsRecalled[dist-1] += chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - # acc = 0 - # charsRecalled_cumulative = [] - # for x in charsRecalled: - # acc += x - # charsRecalled_cumulative.append(acc) - # charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - # max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - # print("% >>> Max avg recalled characters = {}".format(max_values[0])) - # plt.plot(x_values, max_values, 'r-') - # legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - # plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def plot_strategies_charsRecalled_prefix_noncummulative(plot_size=50, selected_strategies=["recent (bash-like)"], show_strat_title=False): - plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance (including prefix matches) (noncummulative) <{}>".format(datetime.now().strftime('%H:%M:%S'))) - plt.ylabel("Average characters recalled (including prefix matches)") - plt.xlabel("Distance") - x_values = range(1, plot_size+1) - legend = [] - saved_charsRecalled_total = None - saved_dataPoint_count = None - for strategy in data["Strategies"]: - strategy_title = strategy["Title"] - # strategy_description = strategy["Description"] - - dataPoint_count = 0 - matches_total = 0 - charsRecalled = [0] * plot_size - charsRecalled_total = 0 - - for multiMatch in strategy["PrefixMatches"]: - dataPoint_count += 1 - - if not multiMatch["Match"]: - continue - matches_total += 1 - - last_charsRecalled = 0 - for match in multiMatch["Entries"]: - - chars = match["CharsRecalled"] - charsIncrease = chars - last_charsRecalled - assert(charsIncrease > 0) - charsRecalled_total += charsIncrease - - dist = match["Distance"] - if dist > plot_size: - continue - - charsRecalled[dist-1] += charsIncrease - last_charsRecalled = chars - - # recent is very simple strategy so we will believe - # that there is no bug in it and we can use it to determine total - if strategy_title == "recent": - saved_charsRecalled_total = charsRecalled_total - saved_dataPoint_count = dataPoint_count - - if len(selected_strategies) and strategy_title not in selected_strategies: - continue - - # acc = 0 - # charsRecalled_cumulative = [] - # for x in charsRecalled: - # acc += x - # charsRecalled_cumulative.append(acc) - # charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) - charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled)) - - plt.plot(x_values, charsRecalled_average, 'o-') - legend.append(strategy_title) - - assert(saved_charsRecalled_total is not None) - assert(saved_dataPoint_count is not None) - # max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) - # print("% >>> Max avg recalled characters (including prefix matches) = {}".format(max_values[0])) - # plt.plot(x_values, max_values, 'r-') - # legend.append("maximum possible") - - x_ticks = list(range(1, plot_size+1, 2)) - x_labels = x_ticks[:] - plt.xticks(x_ticks, x_labels) - # plt.ylim(bottom=0) - if show_strat_title: - plt.legend(legend, loc="best") - if async_draw: - plt.draw() - else: - plt.show() - - -def print_top_cmds(num_cmds=20): - cmd_count = defaultdict(int) - cmd_total = 0 - for pid, session in DATA_records_by_session.items(): - for record in session: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - cmd_total += 1 - - # get `node_count` of largest nodes - sorted_cmd_count = list(sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)) - print("\n\n% All subjects: Top commands") - for cmd, count in sorted_cmd_count[:num_cmds]: - print("{} {}".format(cmd, count)) - # print(sorted_cmd_count) - # cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:cmd_count] - - -def print_top_cmds_by_user(num_cmds=20): - for user in DATA_records_by_user.items(): - name, records = user - cmd_count = defaultdict(int) - cmd_total = 0 - for record in records: - cmd = record["command"] - if cmd == "": - continue - cmd_count[cmd] += 1 - cmd_total += 1 - - # get `node_count` of largest nodes - sorted_cmd_count = list(sorted(cmd_count.items(), key=lambda x: x[1], reverse=True)) - print("\n\n% {}: Top commands".format(name)) - for cmd, count in sorted_cmd_count[:num_cmds]: - print("{} {}".format(cmd, count)) - # print(sorted_cmd_count) - # cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:cmd_count] - - -def print_avg_cmdline_length(): - cmd_len_total = 0 - cmd_total = 0 - for pid, session in DATA_records_by_session.items(): - for record in session: - cmd = record["cmdLine"] - if cmd == "": - continue - cmd_len_total += len(cmd) - cmd_total += 1 - - print("% ALL avg cmdline = {}".format(cmd_len_total / cmd_total)) - # print(sorted_cmd_count) - # cmds_to_graph = list(map(lambda x: x[0], sorted_cmd_count))[:cmd_count] - - -# plot_cmdLineFrq_rank() -# plot_cmdFrq_rank() -print_top_cmds(30) -print_top_cmds_by_user(30) -# print_avg_cmdline_length() -# -# plot_cmdLineVocabularySize_cmdLinesEntered() -plot_cmdVocabularySize_cmdLinesEntered() -plot_cmdVocabularySize_time() -# plot_cmdVocabularySize_daily() -plot_cmdUsage_in_time(num_cmds=100) -plot_cmdUsage_in_time(sort_cmds=True, num_cmds=100) -# -recent_strats=("recent", "recent (bash-like)") -recurrence_strat=("recent (bash-like)",) -# plot_strategies_matches(20) -# plot_strategies_charsRecalled(20) -# plot_strategies_charsRecalled_prefix(20) -# plot_strategies_charsRecalled_noncummulative(20, selected_strategies=recent_strats) -# plot_strategies_matches_noncummulative(20) -# plot_strategies_charsRecalled_noncummulative(20) -# plot_strategies_charsRecalled_prefix_noncummulative(20) -# plot_strategies_matches(20, selected_strategies=recurrence_strat, show_strat_title=True, force_strat_title="recurrence rate") -# plot_strategies_matches_noncummulative(20, selected_strategies=recurrence_strat, show_strat_title=True, force_strat_title="recurrence rate") - -# graph_cmdSequences(node_count=33, edge_minValue=0.048) - -# graph_cmdSequences(node_count=28, edge_minValue=0.06) - -# new improved -# for n in range(40, 43): -# for e in range(94, 106, 2): -# e *= 0.001 -# graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False) - -#for n in range(29, 35): -# for e in range(44, 56, 2): -# e *= 0.001 -# graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False) - -# be careful and check if labels fit the display - -if async_draw: - plt.show() diff --git a/scripts/shellrc.sh b/scripts/shellrc.sh index 311ed2a..2bc84ee 100644 --- a/scripts/shellrc.sh +++ b/scripts/shellrc.sh @@ -47,24 +47,9 @@ __RESH_TERM="$TERM" # non-posix __RESH_RT_SESSION=$(__resh_get_epochrealtime) -__RESH_OSTYPE="$OSTYPE" +__RESH_OSTYPE="$OSTYPE" __RESH_MACHTYPE="$MACHTYPE" -if [ $__RESH_LINUX -eq 1 ]; then - __RESH_OS_RELEASE_ID=$(. /etc/os-release; echo "$ID") - __RESH_OS_RELEASE_VERSION_ID=$(. /etc/os-release; echo "$VERSION_ID") - __RESH_OS_RELEASE_ID_LIKE=$(. /etc/os-release; echo "$ID_LIKE") - __RESH_OS_RELEASE_NAME=$(. /etc/os-release; echo "$NAME") - __RESH_OS_RELEASE_PRETTY_NAME=$(. /etc/os-release; echo "$PRETTY_NAME") - __RESH_RT_SESS_SINCE_BOOT=$(cut -d' ' -f1 /proc/uptime) -elif [ $__RESH_MACOS -eq 1 ]; then - __RESH_OS_RELEASE_ID="macos" - __RESH_OS_RELEASE_VERSION_ID=$(sw_vers -productVersion 2>/dev/null) - __RESH_OS_RELEASE_NAME="macOS" - __RESH_OS_RELEASE_PRETTY_NAME="Mac OS X" - __RESH_RT_SESS_SINCE_BOOT=$(sysctl -n kern.boottime | awk '{print $4}' | sed 's/,//g') -fi - # shellcheck disable=2155 export __RESH_VERSION=$(resh-collect -version) # shellcheck disable=2155 diff --git a/scripts/util.sh b/scripts/util.sh index b434ecc..7383e57 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -68,18 +68,10 @@ __resh_bash_completion_init() { # skip completion init if they are not _get_comp_words_by_ref >/dev/null 2>/dev/null [[ $? == 127 ]] && return - local bash_completion_dir=~/.resh/bash_completion.d - if [[ -d $bash_completion_dir && -r $bash_completion_dir && \ - -x $bash_completion_dir ]]; then - for i in $(LC_ALL=C command ls "$bash_completion_dir"); do - i=$bash_completion_dir/$i - # shellcheck disable=SC2154 - # shellcheck source=/dev/null - [[ -f "$i" && -r "$i" ]] && . "$i" - done - fi + . ~/.resh/bash_completion.d/_reshctl } +// TODO: redo this __resh_zsh_completion_init() { # NOTE: this is hacky - each completion needs to be added individually # TODO: fix later @@ -137,7 +129,6 @@ __resh_session_init() { fi fi if [ "$__RESH_VERSION" = "$(resh-session-init -version)" ] && [ "$__RESH_REVISION" = "$(resh-session-init -revision)" ]; then - local fpath_last_run="$__RESH_XDG_CACHE_HOME/session_init_last_run_out.txt" resh-session-init -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ -shell "$__RESH_SHELL" \ @@ -166,26 +157,6 @@ __resh_session_init() { -osReleaseVersionId "$__RESH_OS_RELEASE_VERSION_ID" \ -osReleaseIdLike "$__RESH_OS_RELEASE_ID_LIKE" \ -osReleaseName "$__RESH_OS_RELEASE_NAME" \ - -osReleasePrettyName "$__RESH_OS_RELEASE_PRETTY_NAME" \ - >| "$fpath_last_run" 2>&1 || echo "resh-session-init ERROR: $(head -n 1 $fpath_last_run)" + -osReleasePrettyName "$__RESH_OS_RELEASE_PRETTY_NAME" fi } - -__resh_set_xdg_home_paths() { - if [ -z "${XDG_CACHE_HOME-}" ]; then - __RESH_XDG_CACHE_HOME="$HOME/.cache/resh" - else - __RESH_XDG_CACHE_HOME="$XDG_CACHE_HOME/resh" - fi - mkdir -p "$__RESH_XDG_CACHE_HOME" >/dev/null 2>/dev/null - export __RESH_XDG_CACHE_HOME - - - if [ -z "${XDG_DATA_HOME-}" ]; then - __RESH_XDG_DATA_HOME="$HOME/.local/share/resh" - else - __RESH_XDG_DATA_HOME="$XDG_DATA_HOME/resh" - fi - mkdir -p "$__RESH_XDG_DATA_HOME" >/dev/null 2>/dev/null - export __RESH_XDG_DATA_HOME -} From 0c6ce3c8e9c1126a7a410a637363556640caacfd Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 18 Oct 2022 02:24:24 +0200 Subject: [PATCH 21/35] get resh to compile and run --- .goreleaser.yml | 9 + Makefile | 2 +- cmd/cli/main.go | 10 +- cmd/collect/main.go | 180 +++----------- cmd/daemon/dump.go | 2 +- cmd/daemon/main.go | 8 +- cmd/daemon/record.go | 16 +- cmd/daemon/run-server.go | 12 +- cmd/daemon/session-init.go | 14 +- cmd/install-utils/main.go | 4 +- cmd/postcollect/main.go | 113 ++------- cmd/session-init/main.go | 145 +---------- go.mod | 1 - go.sum | 2 - internal/cfg/cfg.go | 1 + internal/collect/collect.go | 33 ++- internal/histcli/histcli.go | 11 +- internal/histfile/histfile.go | 127 +++++----- internal/msg/msg.go | 4 +- internal/output/output.go | 20 +- internal/recconv/recconv.go | 28 ++- internal/recio/read.go | 2 +- internal/recio/write.go | 38 ++- internal/record/v1.go | 10 +- internal/recordint/collect.go | 13 + internal/recordint/enriched.go | 51 ---- internal/recordint/searchapp.go | 53 +++- internal/records/records.go | 256 -------------------- internal/records/records_test.go | 85 ------- internal/records/testdata/resh_history.json | 27 --- internal/recutil/recutil.go | 109 +++------ internal/searchapp/item.go | 22 +- internal/searchapp/test.go | 12 +- internal/sesswatch/sesswatch.go | 26 +- scripts/hooks.sh | 75 +----- scripts/install.sh | 5 +- scripts/reshctl.sh | 4 +- scripts/shellrc.sh | 3 +- scripts/util.sh | 42 +--- scripts/widgets.sh | 6 +- 40 files changed, 447 insertions(+), 1134 deletions(-) delete mode 100644 internal/recordint/enriched.go delete mode 100644 internal/records/records_test.go delete mode 100644 internal/records/testdata/resh_history.json diff --git a/.goreleaser.yml b/.goreleaser.yml index c2e2bf5..4cc663d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -102,6 +102,15 @@ builds: - amd64 - arm - arm64 + - + id: "install-utils" + main: ./cmd/install-utils + binary: bin/resh-install-utils + goarch: + - 386 + - amd64 + - arm + - arm64 # signs: # - artifacts: checksum diff --git a/Makefile b/Makefile index 249cd43..e5b5c87 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ GOFLAGS=-ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.de build: submodules bin/resh-session-init bin/resh-collect bin/resh-postcollect\ bin/resh-daemon bin/resh-control bin/resh-config bin/resh-cli\ - bin/installutil + bin/resh-install-utils install: build scripts/install.sh diff --git a/cmd/cli/main.go b/cmd/cli/main.go index fbd0219..5d588f9 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -19,7 +19,7 @@ import ( "github.com/curusarn/resh/internal/logger" "github.com/curusarn/resh/internal/msg" "github.com/curusarn/resh/internal/output" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "github.com/curusarn/resh/internal/searchapp" "go.uber.org/zap" @@ -96,7 +96,7 @@ func runReshCli(out *output.Output, config cfg.Config) (string, int) { st := state{ // lock sync.Mutex - cliRecords: resp.CliRecords, + cliRecords: resp.Records, initialQuery: *query, } @@ -106,7 +106,7 @@ func runReshCli(out *output.Output, config cfg.Config) (string, int) { sessionID: *sessionID, host: *host, pwd: *pwd, - gitOriginRemote: records.NormalizeGitRemote(*gitOriginRemote), + gitOriginRemote: *gitOriginRemote, s: &st, } g.SetManager(layout) @@ -159,7 +159,7 @@ func runReshCli(out *output.Output, config cfg.Config) (string, int) { type state struct { lock sync.Mutex - cliRecords []records.CliRecord + cliRecords []recordint.SearchApp data []searchapp.Item rawData []searchapp.RawItem highlightedItem int @@ -600,7 +600,7 @@ func SendCliMsg(out *output.Output, m msg.CliMsg, port string) msg.CliResponse { out.Fatal("Failed decode response", err) } sugar.Debugw("Recieved records from daemon", - "recordCount", len(response.CliRecords), + "recordCount", len(response.Records), ) return response } diff --git a/cmd/collect/main.go b/cmd/collect/main.go index aac282c..7f19e0a 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -9,7 +9,8 @@ import ( "github.com/curusarn/resh/internal/collect" "github.com/curusarn/resh/internal/logger" "github.com/curusarn/resh/internal/output" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recordint" "go.uber.org/zap" // "os/exec" @@ -32,14 +33,6 @@ func main() { } out := output.New(logger, "resh-collect ERROR") - homeDir, err := os.UserHomeDir() - if err != nil { - out.Fatal("Could not get user home dir", err) - } - - reshUUIDPath := filepath.Join(homeDir, "/.resh/resh-uuid") - machineIDPath := "/etc/machine-id" - // version showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") @@ -49,55 +42,27 @@ func main() { // core cmdLine := flag.String("cmdLine", "", "command line") - exitCode := flag.Int("exitCode", -1, "exit code") - shell := flag.String("shell", "", "actual shell") - uname := flag.String("uname", "", "uname") - sessionID := flag.String("sessionId", "", "resh generated session id") - recordID := flag.String("recordId", "", "resh generated record id") - - // posix variables - cols := flag.String("cols", "-1", "$COLUMNS") - lines := flag.String("lines", "-1", "$LINES") + home := flag.String("home", "", "$HOME") - lang := flag.String("lang", "", "$LANG") - lcAll := flag.String("lcAll", "", "$LC_ALL") - login := flag.String("login", "", "$LOGIN") - // path := flag.String("path", "", "$PATH") pwd := flag.String("pwd", "", "$PWD - present working directory") - shellEnv := flag.String("shellEnv", "", "$SHELL") - term := flag.String("term", "", "$TERM") + + // FIXME: get device ID + deviceID := flag.String("deviceID", "", "RESH device ID") + sessionID := flag.String("sessionID", "", "resh generated session ID") + recordID := flag.String("recordID", "", "resh generated record ID") + sessionPID := flag.Int("sessionPID", -1, "PID at the start of the terminal session") + + shell := flag.String("shell", "", "current shell") + + logname := flag.String("logname", "", "$LOGNAME") + hostname := flag.String("hostname", "", "$HOSTNAME") // non-posix - pid := flag.Int("pid", -1, "$$") - sessionPid := flag.Int("sessionPid", -1, "$$ at session start") shlvl := flag.Int("shlvl", -1, "$SHLVL") - host := flag.String("host", "", "$HOSTNAME") - hosttype := flag.String("hosttype", "", "$HOSTTYPE") - ostype := flag.String("ostype", "", "$OSTYPE") - machtype := flag.String("machtype", "", "$MACHTYPE") - gitCdup := flag.String("gitCdup", "", "git rev-parse --show-cdup") gitRemote := flag.String("gitRemote", "", "git remote get-url origin") - gitCdupExitCode := flag.Int("gitCdupExitCode", -1, "... $?") - gitRemoteExitCode := flag.Int("gitRemoteExitCode", -1, "... $?") - - // before after - timezoneBefore := flag.String("timezoneBefore", "", "") - - osReleaseID := flag.String("osReleaseId", "", "/etc/os-release ID") - osReleaseVersionID := flag.String("osReleaseVersionId", "", - "/etc/os-release ID") - osReleaseIDLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") - osReleaseName := flag.String("osReleaseName", "", "/etc/os-release ID") - osReleasePrettyName := flag.String("osReleasePrettyName", "", - "/etc/os-release ID") - - rtb := flag.String("realtimeBefore", "-1", "before $EPOCHREALTIME") - rtsess := flag.String("realtimeSession", "-1", - "on session start $EPOCHREALTIME") - rtsessboot := flag.String("realtimeSessSinceBoot", "-1", - "on session start $EPOCHREALTIME") + time_ := flag.String("time", "-1", "$EPOCHREALTIME") flag.Parse() if *showVersion == true { @@ -109,37 +74,17 @@ func main() { os.Exit(0) } if *requireVersion != "" && *requireVersion != version { - fmt.Println("Please restart/reload this terminal session " + - "(resh version: " + version + - "; resh version of this terminal session: " + *requireVersion + - ")") - os.Exit(3) + out.FatalVersionMismatch(version, *requireVersion) } if *requireRevision != "" && *requireRevision != commit { - fmt.Println("Please restart/reload this terminal session " + - "(resh revision: " + commit + - "; resh revision of this terminal session: " + *requireRevision + - ")") - os.Exit(3) + // this is only relevant for dev versions so we can reuse FatalVersionMismatch() + out.FatalVersionMismatch("revision "+commit, "revision "+*requireVersion) } - realtimeBefore, err := strconv.ParseFloat(*rtb, 64) - if err != nil { - out.Fatal("Error while parsing flag --realtimeBefore", err) - } - realtimeSessionStart, err := strconv.ParseFloat(*rtsess, 64) + time, err := strconv.ParseFloat(*time_, 64) if err != nil { - out.Fatal("Error while parsing flag --realtimeSession", err) + out.Fatal("Error while parsing flag --time", err) } - realtimeSessSinceBoot, err := strconv.ParseFloat(*rtsessboot, 64) - if err != nil { - out.Fatal("Error while parsing flag --realtimeSessSinceBoot", err) - } - realtimeSinceSessionStart := realtimeBefore - realtimeSessionStart - realtimeSinceBoot := realtimeSessSinceBoot + realtimeSinceSessionStart - - timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(logger, *timezoneBefore) - realtimeBeforeLocal := realtimeBefore + timezoneBeforeOffset realPwd, err := filepath.EvalSymlinks(*pwd) if err != nil { @@ -147,79 +92,34 @@ func main() { realPwd = "" } - gitDir, gitRealDir := collect.GetGitDirs(logger, *gitCdup, *gitCdupExitCode, *pwd) - if *gitRemoteExitCode != 0 { - *gitRemote = "" - } + rec := recordint.Collect{ + SessionID: *sessionID, + Shlvl: *shlvl, + SessionPID: *sessionPID, + + Shell: *shell, - // if *osReleaseID == "" { - // *osReleaseID = "linux" - // } - // if *osReleaseName == "" { - // *osReleaseName = "Linux" - // } - // if *osReleasePrettyName == "" { - // *osReleasePrettyName = "Linux" - // } - - rec := records.Record{ - // posix - Cols: *cols, - Lines: *lines, - // core - BaseRecord: records.BaseRecord{ - CmdLine: *cmdLine, - ExitCode: *exitCode, - Shell: *shell, - Uname: *uname, + Rec: record.V1{ + DeviceID: *deviceID, SessionID: *sessionID, RecordID: *recordID, + CmdLine: *cmdLine, + // posix - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - Pwd: *pwd, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - RealPwd: realPwd, - Pid: *pid, - SessionPID: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - - RealtimeBefore: realtimeBefore, - RealtimeBeforeLocal: realtimeBeforeLocal, - - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - GitDir: gitDir, - GitRealDir: gitRealDir, - GitOriginRemote: *gitRemote, - MachineID: collect.ReadFileContent(out.Logger, machineIDPath), + Home: *home, + Pwd: *pwd, + RealPwd: realPwd, - OsReleaseID: *osReleaseID, - OsReleaseVersionID: *osReleaseVersionID, - OsReleaseIDLike: *osReleaseIDLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, + Logname: *logname, + Hostname: *hostname, + + GitOriginRemote: *gitRemote, - PartOne: true, + Time: time, - ReshUUID: collect.ReadFileContent(out.Logger, reshUUIDPath), - ReshVersion: version, - ReshRevision: commit, + PartOne: true, + PartsNotMerged: true, }, } collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/record") diff --git a/cmd/daemon/dump.go b/cmd/daemon/dump.go index c038705..b3a154f 100644 --- a/cmd/daemon/dump.go +++ b/cmd/daemon/dump.go @@ -40,7 +40,7 @@ func (h *dumpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { sugar.Errorw("Error when getting records", "error", err) } - resp := msg.CliResponse{CliRecords: fullRecords.List} + resp := msg.CliResponse{Records: fullRecords.List} jsn, err = json.Marshal(&resp) if err != nil { sugar.Errorw("Error when marshaling", "error", err) diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index 9338d0b..6a6feff 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -22,7 +22,10 @@ var developement bool func main() { config, errCfg := cfg.New() - logger, _ := logger.New("daemon", config.LogLevel, developement) + logger, err := logger.New("daemon", config.LogLevel, developement) + if err != nil { + fmt.Printf("Error while creating logger: %v", err) + } defer logger.Sync() // flushes buffer, if any if errCfg != nil { logger.Error("Error while getting configuration", zap.Error(errCfg)) @@ -34,9 +37,6 @@ func main() { "commit", commit, ) - // xdgCacheHome := d.getEnvOrPanic("__RESH_XDG_CACHE_HOME") - // xdgDataHome := d.getEnvOrPanic("__RESH_XDG_DATA_HOME") - // TODO: rethink PID file and logs location homeDir, err := os.UserHomeDir() if err != nil { diff --git a/cmd/daemon/record.go b/cmd/daemon/record.go index e9acfcc..34e9c1c 100644 --- a/cmd/daemon/record.go +++ b/cmd/daemon/record.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "net/http" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "go.uber.org/zap" ) -func NewRecordHandler(sugar *zap.SugaredLogger, subscribers []chan records.Record) recordHandler { +func NewRecordHandler(sugar *zap.SugaredLogger, subscribers []chan recordint.Collect) recordHandler { return recordHandler{ sugar: sugar.With(zap.String("endpoint", "/record")), subscribers: subscribers, @@ -18,7 +18,7 @@ func NewRecordHandler(sugar *zap.SugaredLogger, subscribers []chan records.Recor type recordHandler struct { sugar *zap.SugaredLogger - subscribers []chan records.Record + subscribers []chan recordint.Collect } func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -34,8 +34,8 @@ func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } sugar.Debugw("Unmarshaling record ...") - record := records.Record{} - err = json.Unmarshal(jsn, &record) + rec := recordint.Collect{} + err = json.Unmarshal(jsn, &rec) if err != nil { sugar.Errorw("Error during unmarshaling", "error", err, @@ -44,16 +44,16 @@ func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } part := "2" - if record.PartOne { + if rec.Rec.PartOne { part = "1" } sugar := sugar.With( - "cmdLine", record.CmdLine, + "cmdLine", rec.Rec.CmdLine, "part", part, ) sugar.Debugw("Got record, sending to subscribers ...") for _, sub := range h.subscribers { - sub <- record + sub <- rec } sugar.Debugw("Record sent to subscribers") }() diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index 7878f9a..00d9ede 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -8,7 +8,7 @@ import ( "github.com/curusarn/resh/internal/cfg" "github.com/curusarn/resh/internal/histfile" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "github.com/curusarn/resh/internal/sesswatch" "github.com/curusarn/resh/internal/signalhandler" "go.uber.org/zap" @@ -26,15 +26,15 @@ type Server struct { } func (s *Server) Run() { - var recordSubscribers []chan records.Record - var sessionInitSubscribers []chan records.Record + var recordSubscribers []chan recordint.Collect + var sessionInitSubscribers []chan recordint.SessionInit var sessionDropSubscribers []chan string var signalSubscribers []chan os.Signal shutdown := make(chan string) // histfile - histfileRecords := make(chan records.Record) + histfileRecords := make(chan recordint.Collect) recordSubscribers = append(recordSubscribers, histfileRecords) histfileSessionsToDrop := make(chan string) sessionDropSubscribers = append(sessionDropSubscribers, histfileSessionsToDrop) @@ -48,9 +48,9 @@ func (s *Server) Run() { histfileSignals, shutdown) // sesswatch - sesswatchRecords := make(chan records.Record) + sesswatchRecords := make(chan recordint.Collect) recordSubscribers = append(recordSubscribers, sesswatchRecords) - sesswatchSessionsToWatch := make(chan records.Record) + sesswatchSessionsToWatch := make(chan recordint.SessionInit) sessionInitSubscribers = append(sessionInitSubscribers, sesswatchSessionsToWatch) sesswatch.Go( s.sugar, diff --git a/cmd/daemon/session-init.go b/cmd/daemon/session-init.go index 2015fb0..9ddd76f 100644 --- a/cmd/daemon/session-init.go +++ b/cmd/daemon/session-init.go @@ -5,13 +5,13 @@ import ( "io/ioutil" "net/http" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "go.uber.org/zap" ) type sessionInitHandler struct { sugar *zap.SugaredLogger - subscribers []chan records.Record + subscribers []chan recordint.SessionInit } func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -28,8 +28,8 @@ func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } sugar.Debugw("Unmarshaling record ...") - record := records.Record{} - err = json.Unmarshal(jsn, &record) + rec := recordint.SessionInit{} + err = json.Unmarshal(jsn, &rec) if err != nil { sugar.Errorw("Error during unmarshaling", "error", err, @@ -38,12 +38,12 @@ func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } sugar := sugar.With( - "sessionID", record.SessionID, - "sessionPID", record.SessionPID, + "sessionID", rec.SessionID, + "sessionPID", rec.SessionPID, ) sugar.Infow("Got session, sending to subscribers ...") for _, sub := range h.subscribers { - sub <- record + sub <- rec } sugar.Debugw("Session sent to subscribers") }() diff --git a/cmd/install-utils/main.go b/cmd/install-utils/main.go index 4e83ec0..6c7209e 100644 --- a/cmd/install-utils/main.go +++ b/cmd/install-utils/main.go @@ -20,7 +20,9 @@ func main() { case "backup": backup() case "rollback": - rollback() + // FIXME + panic("Rollback not implemented yet!") + // rollback() case "migrate-config": migrateConfig() case "migrate-history": diff --git a/cmd/postcollect/main.go b/cmd/postcollect/main.go index 4d4fca8..41a3172 100644 --- a/cmd/postcollect/main.go +++ b/cmd/postcollect/main.go @@ -9,12 +9,12 @@ import ( "github.com/curusarn/resh/internal/collect" "github.com/curusarn/resh/internal/logger" "github.com/curusarn/resh/internal/output" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recordint" "go.uber.org/zap" // "os/exec" - "path/filepath" "strconv" ) @@ -32,44 +32,20 @@ func main() { } out := output.New(logger, "resh-postcollect ERROR") - homeDir, err := os.UserHomeDir() - if err != nil { - out.Fatal("Could not get user home dir", err) - } - reshUUIDPath := filepath.Join(homeDir, "/.resh/resh-uuid") - machineIDPath := "/etc/machine-id" - showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") requireVersion := flag.String("requireVersion", "", "abort if version doesn't match") requireRevision := flag.String("requireRevision", "", "abort if revision doesn't match") - cmdLine := flag.String("cmdLine", "", "command line") exitCode := flag.Int("exitCode", -1, "exit code") - sessionID := flag.String("sessionId", "", "resh generated session id") - recordID := flag.String("recordId", "", "resh generated record id") + sessionID := flag.String("sessionID", "", "resh generated session id") + recordID := flag.String("recordID", "", "resh generated record id") shlvl := flag.Int("shlvl", -1, "$SHLVL") - shell := flag.String("shell", "", "actual shell") - - // posix variables - pwdAfter := flag.String("pwdAfter", "", "$PWD after command") - - // non-posix - // sessionPid := flag.Int("sessionPid", -1, "$$ at session start") - - gitCdupAfter := flag.String("gitCdupAfter", "", "git rev-parse --show-cdup") - gitRemoteAfter := flag.String("gitRemoteAfter", "", "git remote get-url origin") - gitCdupExitCodeAfter := flag.Int("gitCdupExitCodeAfter", -1, "... $?") - gitRemoteExitCodeAfter := flag.Int("gitRemoteExitCodeAfter", -1, "... $?") - - // before after - timezoneAfter := flag.String("timezoneAfter", "", "") - - rtb := flag.String("realtimeBefore", "-1", "before $EPOCHREALTIME") - rta := flag.String("realtimeAfter", "-1", "after $EPOCHREALTIME") + rtb := flag.String("timeBefore", "-1", "before $EPOCHREALTIME") + rta := flag.String("timeAfter", "-1", "after $EPOCHREALTIME") flag.Parse() if *showVersion == true { @@ -81,77 +57,36 @@ func main() { os.Exit(0) } if *requireVersion != "" && *requireVersion != version { - fmt.Println("Please restart/reload this terminal session " + - "(resh version: " + version + - "; resh version of this terminal session: " + *requireVersion + - ")") - os.Exit(3) + out.FatalVersionMismatch(version, *requireVersion) } if *requireRevision != "" && *requireRevision != commit { - fmt.Println("Please restart/reload this terminal session " + - "(resh revision: " + commit + - "; resh revision of this terminal session: " + *requireRevision + - ")") - os.Exit(3) + // this is only relevant for dev versions so we can reuse FatalVersionMismatch() + out.FatalVersionMismatch("revision "+commit, "revision "+*requireVersion) } - realtimeAfter, err := strconv.ParseFloat(*rta, 64) - if err != nil { - out.Fatal("Error while parsing flag --realtimeAfter", err) - } - realtimeBefore, err := strconv.ParseFloat(*rtb, 64) + + timeAfter, err := strconv.ParseFloat(*rta, 64) if err != nil { - out.Fatal("Error while parsing flag --realtimeBefore", err) + out.Fatal("Error while parsing flag --timeAfter", err) } - realtimeDuration := realtimeAfter - realtimeBefore - - timezoneAfterOffset := collect.GetTimezoneOffsetInSeconds(logger, *timezoneAfter) - realtimeAfterLocal := realtimeAfter + timezoneAfterOffset - - realPwdAfter, err := filepath.EvalSymlinks(*pwdAfter) + timeBefore, err := strconv.ParseFloat(*rtb, 64) if err != nil { - logger.Error("Error while handling pwdAfter realpath", zap.Error(err)) - realPwdAfter = "" + out.Fatal("Error while parsing flag --timeBefore", err) } + duration := timeAfter - timeBefore - gitDirAfter, gitRealDirAfter := collect.GetGitDirs(logger, *gitCdupAfter, *gitCdupExitCodeAfter, *pwdAfter) - if *gitRemoteExitCodeAfter != 0 { - *gitRemoteAfter = "" - } + // FIXME: use recordint.Postollect + rec := recordint.Collect{ + SessionID: *sessionID, + Shlvl: *shlvl, - rec := records.Record{ - // core - BaseRecord: records.BaseRecord{ - CmdLine: *cmdLine, - ExitCode: *exitCode, - SessionID: *sessionID, + Rec: record.V1{ RecordID: *recordID, - Shlvl: *shlvl, - Shell: *shell, - - PwdAfter: *pwdAfter, - - // non-posix - RealPwdAfter: realPwdAfter, - - // before after - TimezoneAfter: *timezoneAfter, - - RealtimeBefore: realtimeBefore, - RealtimeAfter: realtimeAfter, - RealtimeAfterLocal: realtimeAfterLocal, - - RealtimeDuration: realtimeDuration, - - GitDirAfter: gitDirAfter, - GitRealDirAfter: gitRealDirAfter, - GitOriginRemoteAfter: *gitRemoteAfter, - MachineID: collect.ReadFileContent(out.Logger, machineIDPath), + SessionID: *sessionID, - PartOne: false, + ExitCode: *exitCode, + Duration: duration, - ReshUUID: collect.ReadFileContent(out.Logger, reshUUIDPath), - ReshVersion: version, - ReshRevision: commit, + PartsNotMerged: true, }, } collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/record") diff --git a/cmd/session-init/main.go b/cmd/session-init/main.go index 59010f7..c966f23 100644 --- a/cmd/session-init/main.go +++ b/cmd/session-init/main.go @@ -9,10 +9,9 @@ import ( "github.com/curusarn/resh/internal/collect" "github.com/curusarn/resh/internal/logger" "github.com/curusarn/resh/internal/output" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "go.uber.org/zap" - "path/filepath" "strconv" ) @@ -30,61 +29,15 @@ func main() { } out := output.New(logger, "resh-collect ERROR") - homeDir, err := os.UserHomeDir() - if err != nil { - out.Fatal("Could not get user home dir", err) - } - - reshUUIDPath := filepath.Join(homeDir, "/.resh/resh-uuid") - - machineIDPath := "/etc/machine-id" - showVersion := flag.Bool("version", false, "Show version and exit") showRevision := flag.Bool("revision", false, "Show git revision and exit") requireVersion := flag.String("requireVersion", "", "abort if version doesn't match") requireRevision := flag.String("requireRevision", "", "abort if revision doesn't match") - shell := flag.String("shell", "", "actual shell") - uname := flag.String("uname", "", "uname") sessionID := flag.String("sessionId", "", "resh generated session id") - // posix variables - cols := flag.String("cols", "-1", "$COLUMNS") - lines := flag.String("lines", "-1", "$LINES") - home := flag.String("home", "", "$HOME") - lang := flag.String("lang", "", "$LANG") - lcAll := flag.String("lcAll", "", "$LC_ALL") - login := flag.String("login", "", "$LOGIN") - shellEnv := flag.String("shellEnv", "", "$SHELL") - term := flag.String("term", "", "$TERM") - - // non-posix - pid := flag.Int("pid", -1, "$$") - sessionPid := flag.Int("sessionPid", -1, "$$ at session start") - shlvl := flag.Int("shlvl", -1, "$SHLVL") - - host := flag.String("host", "", "$HOSTNAME") - hosttype := flag.String("hosttype", "", "$HOSTTYPE") - ostype := flag.String("ostype", "", "$OSTYPE") - machtype := flag.String("machtype", "", "$MACHTYPE") - - // before after - timezoneBefore := flag.String("timezoneBefore", "", "") - - osReleaseID := flag.String("osReleaseId", "", "/etc/os-release ID") - osReleaseVersionID := flag.String("osReleaseVersionId", "", - "/etc/os-release ID") - osReleaseIDLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") - osReleaseName := flag.String("osReleaseName", "", "/etc/os-release ID") - osReleasePrettyName := flag.String("osReleasePrettyName", "", - "/etc/os-release ID") - - rtb := flag.String("realtimeBefore", "-1", "before $EPOCHREALTIME") - rtsess := flag.String("realtimeSession", "-1", - "on session start $EPOCHREALTIME") - rtsessboot := flag.String("realtimeSessSinceBoot", "-1", - "on session start $EPOCHREALTIME") + sessionPID := flag.Int("sessionPid", -1, "$$ at session start") flag.Parse() if *showVersion == true { @@ -96,96 +49,16 @@ func main() { os.Exit(0) } if *requireVersion != "" && *requireVersion != version { - fmt.Println("Please restart/reload this terminal session " + - "(resh version: " + version + - "; resh version of this terminal session: " + *requireVersion + - ")") - os.Exit(3) + out.FatalVersionMismatch(version, *requireVersion) } if *requireRevision != "" && *requireRevision != commit { - fmt.Println("Please restart/reload this terminal session " + - "(resh revision: " + commit + - "; resh revision of this terminal session: " + *requireRevision + - ")") - os.Exit(3) - } - realtimeBefore, err := strconv.ParseFloat(*rtb, 64) - if err != nil { - out.Fatal("Error while parsing flag --realtimeBefore", err) + // this is only relevant for dev versions so we can reuse FatalVersionMismatch() + out.FatalVersionMismatch("revision "+commit, "revision "+*requireVersion) } - realtimeSessionStart, err := strconv.ParseFloat(*rtsess, 64) - if err != nil { - out.Fatal("Error while parsing flag --realtimeSession", err) - } - realtimeSessSinceBoot, err := strconv.ParseFloat(*rtsessboot, 64) - if err != nil { - out.Fatal("Error while parsing flag --realtimeSessSinceBoot", err) - } - realtimeSinceSessionStart := realtimeBefore - realtimeSessionStart - realtimeSinceBoot := realtimeSessSinceBoot + realtimeSinceSessionStart - - timezoneBeforeOffset := collect.GetTimezoneOffsetInSeconds(logger, *timezoneBefore) - realtimeBeforeLocal := realtimeBefore + timezoneBeforeOffset - - if *osReleaseID == "" { - *osReleaseID = "linux" - } - if *osReleaseName == "" { - *osReleaseName = "Linux" - } - if *osReleasePrettyName == "" { - *osReleasePrettyName = "Linux" - } - - rec := records.Record{ - // posix - Cols: *cols, - Lines: *lines, - // core - BaseRecord: records.BaseRecord{ - Shell: *shell, - Uname: *uname, - SessionID: *sessionID, - - // posix - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - Pid: *pid, - SessionPID: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - - RealtimeBefore: realtimeBefore, - RealtimeBeforeLocal: realtimeBeforeLocal, - - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - MachineID: collect.ReadFileContent(out.Logger, machineIDPath), - - OsReleaseID: *osReleaseID, - OsReleaseVersionID: *osReleaseVersionID, - OsReleaseIDLike: *osReleaseIDLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, - ReshUUID: collect.ReadFileContent(out.Logger, reshUUIDPath), - ReshVersion: version, - ReshRevision: commit, - }, + rec := recordint.SessionInit{ + SessionID: *sessionID, + SessionPID: *sessionPID, } - collect.SendRecord(out, rec, strconv.Itoa(config.Port), "/session_init") + collect.SendSessionInit(out, rec, strconv.Itoa(config.Port)) } diff --git a/go.mod b/go.mod index c63b38f..db1e19f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.18 require ( github.com/BurntSushi/toml v0.4.1 github.com/awesome-gocui/gocui v1.0.0 - github.com/mattn/go-shellwords v1.0.12 github.com/mitchellh/go-ps v1.0.0 github.com/spf13/cobra v1.2.1 github.com/whilp/git-urls v1.0.0 diff --git a/go.sum b/go.sum index 2e9120d..ec1b8bf 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 99f7c84..4df3d94 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -92,6 +92,7 @@ func getConfig() (*configFile, error) { return readConfig(path) } +// returned config is always usable, returned errors are informative func processAndFillDefaults(configF *configFile) (Config, error) { config := defaults diff --git a/internal/collect/collect.go b/internal/collect/collect.go index e9fc585..15d2bbb 100644 --- a/internal/collect/collect.go +++ b/internal/collect/collect.go @@ -11,14 +11,14 @@ import ( "time" "github.com/curusarn/resh/internal/output" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "go.uber.org/zap" ) // SendRecord to daemon -func SendRecord(out *output.Output, r records.Record, port, path string) { +func SendRecord(out *output.Output, r recordint.Collect, port, path string) { out.Logger.Debug("Sending record ...", - zap.String("cmdLine", r.CmdLine), + zap.String("cmdLine", r.Rec.CmdLine), zap.String("sessionID", r.SessionID), ) recJSON, err := json.Marshal(r) @@ -42,6 +42,33 @@ func SendRecord(out *output.Output, r records.Record, port, path string) { } } +// SendSessionInit to daemon +func SendSessionInit(out *output.Output, r recordint.SessionInit, port string) { + out.Logger.Debug("Sending session init ...", + zap.String("sessionID", r.SessionID), + zap.Int("sessionPID", r.SessionPID), + ) + recJSON, err := json.Marshal(r) + if err != nil { + out.Fatal("Error while encoding record", err) + } + + req, err := http.NewRequest("POST", "http://localhost:"+port+"/session_init", + bytes.NewBuffer(recJSON)) + if err != nil { + out.Fatal("Error while sending record", err) + } + req.Header.Set("Content-Type", "application/json") + + client := http.Client{ + Timeout: 1 * time.Second, + } + _, err = client.Do(req) + if err != nil { + out.FatalDaemonNotRunning(err) + } +} + // ReadFileContent and return it as a string func ReadFileContent(logger *zap.Logger, path string) string { dat, err := ioutil.ReadFile(path) diff --git a/internal/histcli/histcli.go b/internal/histcli/histcli.go index 9bedf61..8069990 100644 --- a/internal/histcli/histcli.go +++ b/internal/histcli/histcli.go @@ -1,13 +1,13 @@ package histcli import ( - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" ) // Histcli is a dump of history preprocessed for resh cli purposes type Histcli struct { // list of records - List []records.CliRecord + List []recordint.SearchApp } // New Histcli @@ -16,16 +16,15 @@ func New() Histcli { } // AddRecord to the histcli -func (h *Histcli) AddRecord(record records.Record) { - enriched := records.Enriched(record) - cli := records.NewCliRecord(enriched) +func (h *Histcli) AddRecord(rec *recordint.Indexed) { + cli := recordint.NewSearchApp(rec) h.List = append(h.List, cli) } // AddCmdLine to the histcli func (h *Histcli) AddCmdLine(cmdline string) { - cli := records.NewCliRecordFromCmdLine(cmdline) + cli := recordint.NewSearchAppFromCmdLine(cmdline) h.List = append(h.List, cli) } diff --git a/internal/histfile/histfile.go b/internal/histfile/histfile.go index 9d7cce0..9eea14d 100644 --- a/internal/histfile/histfile.go +++ b/internal/histfile/histfile.go @@ -1,7 +1,6 @@ package histfile import ( - "encoding/json" "math" "os" "strconv" @@ -9,42 +8,48 @@ import ( "github.com/curusarn/resh/internal/histcli" "github.com/curusarn/resh/internal/histlist" + "github.com/curusarn/resh/internal/recio" + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recordint" "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recutil" "go.uber.org/zap" ) +// TODO: get rid of histfile - use histio instead // Histfile writes records to histfile type Histfile struct { sugar *zap.SugaredLogger sessionsMutex sync.Mutex - sessions map[string]records.Record + sessions map[string]recordint.Collect historyPath string - recentMutex sync.Mutex - recentRecords []records.Record - // NOTE: we have separate histories which only differ if there was not enough resh_history // resh_history itself is common for both bash and zsh bashCmdLines histlist.Histlist zshCmdLines histlist.Histlist cliRecords histcli.Histcli + + rio *recio.RecIO } // New creates new histfile and runs its gorutines -func New(sugar *zap.SugaredLogger, input chan records.Record, sessionsToDrop chan string, +func New(sugar *zap.SugaredLogger, input chan recordint.Collect, sessionsToDrop chan string, reshHistoryPath string, bashHistoryPath string, zshHistoryPath string, maxInitHistSize int, minInitHistSizeKB int, signals chan os.Signal, shutdownDone chan string) *Histfile { + rio := recio.New(sugar.With("module", "histfile")) hf := Histfile{ sugar: sugar.With("module", "histfile"), - sessions: map[string]records.Record{}, + sessions: map[string]recordint.Collect{}, historyPath: reshHistoryPath, bashCmdLines: histlist.New(sugar), zshCmdLines: histlist.New(sugar), cliRecords: histcli.New(), + rio: &rio, } go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB) go hf.writer(input, signals, shutdownDone) @@ -53,7 +58,7 @@ func New(sugar *zap.SugaredLogger, input chan records.Record, sessionsToDrop cha } // load records from resh history, reverse, enrich and save -func (h *Histfile) loadCliRecords(recs []records.Record) { +func (h *Histfile) loadCliRecords(recs []recordint.Indexed) { for _, cmdline := range h.bashCmdLines.List { h.cliRecords.AddCmdLine(cmdline) } @@ -62,7 +67,7 @@ func (h *Histfile) loadCliRecords(recs []records.Record) { } for i := len(recs) - 1; i >= 0; i-- { rec := recs[i] - h.cliRecords.AddRecord(rec) + h.cliRecords.AddRecord(&rec) } h.sugar.Infow("Resh history loaded", "historyRecordsCount", len(h.cliRecords.List), @@ -71,8 +76,6 @@ func (h *Histfile) loadCliRecords(recs []records.Record) { // loadsHistory from resh_history and if there is not enough of it also load native shell histories func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHistSize, minInitHistSizeKB int) { - h.recentMutex.Lock() - defer h.recentMutex.Unlock() h.sugar.Infow("Checking if resh_history is large enough ...") fi, err := os.Stat(h.historyPath) var size int @@ -95,14 +98,17 @@ func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHi h.sugar.Debugw("Loading resh history from file ...", "historyFile", h.historyPath, ) - history := records.LoadFromFile(h.sugar, h.historyPath) + history, err := h.rio.ReadAndFixFile(h.historyPath, 3) + if err != nil { + h.sugar.Panicf("Failed to read file: %w", err) + } h.sugar.Infow("Resh history loaded from file", "historyFile", h.historyPath, "recordCount", len(history), ) go h.loadCliRecords(history) // NOTE: keeping this weird interface for now because we might use it in the future - // when we only load bash or zsh history + // when we only load bash or zsh history reshCmdLines := loadCmdLines(h.sugar, history) h.sugar.Infow("Resh history loaded and processed", "recordCount", len(reshCmdLines.List), @@ -130,7 +136,7 @@ func (h *Histfile) sessionGC(sessionsToDrop chan string) { if part1, found := h.sessions[session]; found == true { sugar.Infow("Dropping session") delete(h.sessions, session) - go writeRecord(sugar, part1, h.historyPath) + go h.rio.AppendToFile(h.historyPath, []record.V1{part1.Rec}) } else { sugar.Infow("No hanging parts for session - nothing to drop") } @@ -139,43 +145,43 @@ func (h *Histfile) sessionGC(sessionsToDrop chan string) { } // writer reads records from channel, merges them and writes them to file -func (h *Histfile) writer(input chan records.Record, signals chan os.Signal, shutdownDone chan string) { +func (h *Histfile) writer(collect chan recordint.Collect, signals chan os.Signal, shutdownDone chan string) { for { func() { select { - case record := <-input: + case rec := <-collect: part := "2" - if record.PartOne { + if rec.Rec.PartOne { part = "1" } sugar := h.sugar.With( - "recordCmdLine", record.CmdLine, + "recordCmdLine", rec.Rec.CmdLine, "recordPart", part, - "recordShell", record.Shell, + "recordShell", rec.Shell, ) sugar.Debugw("Got record") h.sessionsMutex.Lock() defer h.sessionsMutex.Unlock() // allows nested sessions to merge records properly - mergeID := record.SessionID + "_" + strconv.Itoa(record.Shlvl) + mergeID := rec.SessionID + "_" + strconv.Itoa(rec.Shlvl) sugar = sugar.With("mergeID", mergeID) - if record.PartOne { + if rec.Rec.PartOne { if _, found := h.sessions[mergeID]; found { msg := "Got another first part of the records before merging the previous one - overwriting!" - if record.Shell == "zsh" { + if rec.Shell == "zsh" { sugar.Warnw(msg) } else { sugar.Infow(msg + " Unfortunately this is normal in bash, it can't be prevented.") } } - h.sessions[mergeID] = record + h.sessions[mergeID] = rec } else { if part1, found := h.sessions[mergeID]; found == false { sugar.Warnw("Got second part of record and nothing to merge it with - ignoring!") } else { delete(h.sessions, mergeID) - go h.mergeAndWriteRecord(sugar, part1, record) + go h.mergeAndWriteRecord(sugar, part1, rec) } } case sig := <-signals: @@ -187,11 +193,11 @@ func (h *Histfile) writer(input chan records.Record, signals chan os.Signal, shu defer h.sessionsMutex.Unlock() sugar.Debugw("Unlocked mutex") - for sessID, record := range h.sessions { + for sessID, rec := range h.sessions { sugar.Warnw("Writing incomplete record for session", "sessionID", sessID, ) - h.writeRecord(sugar, record) + h.writeRecord(sugar, rec.Rec) } sugar.Debugw("Shutdown successful") shutdownDone <- "histfile" @@ -201,52 +207,53 @@ func (h *Histfile) writer(input chan records.Record, signals chan os.Signal, shu } } -func (h *Histfile) writeRecord(sugar *zap.SugaredLogger, part1 records.Record) { - writeRecord(sugar, part1, h.historyPath) +func (h *Histfile) writeRecord(sugar *zap.SugaredLogger, rec record.V1) { + h.rio.AppendToFile(h.historyPath, []record.V1{rec}) } -func (h *Histfile) mergeAndWriteRecord(sugar *zap.SugaredLogger, part1, part2 records.Record) { - err := part1.Merge(part2) +func (h *Histfile) mergeAndWriteRecord(sugar *zap.SugaredLogger, part1 recordint.Collect, part2 recordint.Collect) { + rec, err := recutil.Merge(&part1, &part2) if err != nil { sugar.Errorw("Error while merging records", "error", err) return } func() { - h.recentMutex.Lock() - defer h.recentMutex.Unlock() - h.recentRecords = append(h.recentRecords, part1) - cmdLine := part1.CmdLine + cmdLine := rec.CmdLine h.bashCmdLines.AddCmdLine(cmdLine) h.zshCmdLines.AddCmdLine(cmdLine) - h.cliRecords.AddRecord(part1) + h.cliRecords.AddRecord(&recordint.Indexed{ + // TODO: is this what we want? + Rec: rec, + }) }() - writeRecord(sugar, part1, h.historyPath) + h.rio.AppendToFile(h.historyPath, []record.V1{rec}) } -func writeRecord(sugar *zap.SugaredLogger, rec records.Record, outputPath string) { - recJSON, err := json.Marshal(rec) - if err != nil { - sugar.Errorw("Marshalling error", "error", err) - return - } - f, err := os.OpenFile(outputPath, - os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - sugar.Errorw("Could not open file", "error", err) - return - } - defer f.Close() - _, err = f.Write(append(recJSON, []byte("\n")...)) - if err != nil { - sugar.Errorw("Error while writing record", - "recordRaw", rec, - "error", err, - ) - return - } -} +// TODO: use errors in RecIO +// func writeRecord(sugar *zap.SugaredLogger, rec record.V1, outputPath string) { +// recJSON, err := json.Marshal(rec) +// if err != nil { +// sugar.Errorw("Marshalling error", "error", err) +// return +// } +// f, err := os.OpenFile(outputPath, +// os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) +// if err != nil { +// sugar.Errorw("Could not open file", "error", err) +// return +// } +// defer f.Close() +// _, err = f.Write(append(recJSON, []byte("\n")...)) +// if err != nil { +// sugar.Errorw("Error while writing record", +// "recordRaw", rec, +// "error", err, +// ) +// return +// } +// } // DumpCliRecords returns enriched records func (h *Histfile) DumpCliRecords() histcli.Histcli { @@ -254,13 +261,13 @@ func (h *Histfile) DumpCliRecords() histcli.Histcli { return h.cliRecords } -func loadCmdLines(sugar *zap.SugaredLogger, recs []records.Record) histlist.Histlist { +func loadCmdLines(sugar *zap.SugaredLogger, recs []recordint.Indexed) histlist.Histlist { hl := histlist.New(sugar) // go from bottom and deduplicate var cmdLines []string cmdLinesSet := map[string]bool{} for i := len(recs) - 1; i >= 0; i-- { - cmdLine := recs[i].CmdLine + cmdLine := recs[i].Rec.CmdLine if cmdLinesSet[cmdLine] { continue } diff --git a/internal/msg/msg.go b/internal/msg/msg.go index 06d87cf..7d3b103 100644 --- a/internal/msg/msg.go +++ b/internal/msg/msg.go @@ -1,5 +1,7 @@ package msg +import "github.com/curusarn/resh/internal/recordint" + // CliMsg struct type CliMsg struct { SessionID string @@ -8,7 +10,7 @@ type CliMsg struct { // CliResponse struct type CliResponse struct { - Records []record.SearchApp + Records []recordint.SearchApp } // StatusResponse struct diff --git a/internal/output/output.go b/internal/output/output.go index 769f099..2799e57 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -39,7 +39,7 @@ func (f *Output) Fatal(msg string, err error) { var msgDeamonNotRunning = `Resh-daemon didn't respond - it's probably not running. -> Try restarting this terminal window to bring resh-daemon back up - -> If the problem persists you can check resh-daemon logs: ~/.resh/log.json + -> If the problem persists you can check resh-daemon logs: ~/.local/share/resh/log.json (or ~/$XDG_DATA_HOME/resh/log.json) -> You can create an issue at: https://github.com/curusarn/resh/issues ` var msgVersionMismatch = `This terminal session was started with different resh version than is installed now. @@ -58,12 +58,18 @@ func (f *Output) FatalDaemonNotRunning(err error) { f.Logger.Fatal("Daemon is not running", zap.Error(err)) } -func (f *Output) ErrorVersionMismatch(err error) { - fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgVersionMismatch) - f.Logger.Fatal("Version mismatch", zap.Error(err)) +func (f *Output) ErrorVersionMismatch(installedVer, terminalVer string) { + fmt.Fprintf(os.Stderr, "%s: %s\n\n(installed version: %s, this terminal version: %s)", + f.ErrPrefix, msgVersionMismatch, installedVer, terminalVer) + f.Logger.Fatal("Version mismatch", + zap.String("installed", installedVer), + zap.String("terminal", terminalVer)) } -func (f *Output) FatalVersionMismatch(err error) { - fmt.Fprintf(os.Stderr, "%s: %s", f.ErrPrefix, msgVersionMismatch) - f.Logger.Fatal("Version mismatch", zap.Error(err)) +func (f *Output) FatalVersionMismatch(installedVer, terminalVer string) { + fmt.Fprintf(os.Stderr, "%s: %s\n\n(installed version: %s, this terminal version: %s)", + f.ErrPrefix, msgVersionMismatch, installedVer, terminalVer) + f.Logger.Fatal("Version mismatch", + zap.String("installed", installedVer), + zap.String("terminal", terminalVer)) } diff --git a/internal/recconv/recconv.go b/internal/recconv/recconv.go index d0156c5..36cf605 100644 --- a/internal/recconv/recconv.go +++ b/internal/recconv/recconv.go @@ -1,9 +1,35 @@ package recconv -import "github.com/curusarn/resh/internal/record" +import ( + "github.com/curusarn/resh/internal/record" +) func LegacyToV1(r *record.Legacy) *record.V1 { return &record.V1{ // FIXME: fill in all the fields + + // Flags: 0, + + DeviceID: r.MachineID, + SessionID: r.SessionID, + RecordID: r.RecordID, + + CmdLine: r.CmdLine, + ExitCode: r.ExitCode, + + Home: r.Home, + Pwd: r.Pwd, + RealPwd: r.RealPwd, + + Logname: r.Login, + Hostname: r.Host, + + GitOriginRemote: r.GitOriginRemote, + + Time: r.RealtimeBefore, + Duration: r.RealtimeDuration, + + PartOne: r.PartOne, + PartsNotMerged: !r.PartsMerged, } } diff --git a/internal/recio/read.go b/internal/recio/read.go index 7a7f183..304a4bd 100644 --- a/internal/recio/read.go +++ b/internal/recio/read.go @@ -48,7 +48,7 @@ func (r *RecIO) ReadAndFixFile(fpath string, maxErrors int) ([]recordint.Indexed for _, rec := range recs { recsV1 = append(recsV1, rec.Rec) } - err = r.WriteFile(fpath, recsV1) + err = r.OverwriteFile(fpath, recsV1) if err != nil { r.sugar.Errorw("Failed write fixed history file - aborting fixing history file", "filename", fpath, diff --git a/internal/recio/write.go b/internal/recio/write.go index 8bc5e06..831a146 100644 --- a/internal/recio/write.go +++ b/internal/recio/write.go @@ -10,13 +10,36 @@ import ( ) // TODO: better errors -func (r *RecIO) WriteFile(fpath string, data []record.V1) error { +func (r *RecIO) OverwriteFile(fpath string, recs []record.V1) error { file, err := os.Create(fpath) if err != nil { return err } defer file.Close() - for _, rec := range data { + return writeRecords(file, recs) +} + +// TODO: better errors +func (r *RecIO) AppendToFile(fpath string, recs []record.V1) error { + file, err := os.OpenFile(fpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + return writeRecords(file, recs) +} + +// TODO: better errors +func (r *RecIO) EditRecordFlagsInFile(fpath string, idx int, rec recordint.Flag) error { + // FIXME: implement + // open file "not as append" + // scan to the correct line + r.sugar.Error("not implemented yet (FIXME)") + return nil +} + +func writeRecords(file *os.File, recs []record.V1) error { + for _, rec := range recs { jsn, err := encodeV1Record(rec) if err != nil { return err @@ -29,18 +52,11 @@ func (r *RecIO) WriteFile(fpath string, data []record.V1) error { return nil } -func (r *RecIO) EditRecordFlagsInFile(fpath string, idx int, rec recordint.Flag) error { - // FIXME: implement - // open file "not as append" - // scan to the correct line - - return nil -} - func encodeV1Record(rec record.V1) ([]byte, error) { + version := []byte("v1") jsn, err := json.Marshal(rec) if err != nil { return nil, fmt.Errorf("failed to encode json: %w", err) } - return append(jsn, []byte("\n")...), nil + return append(append(version, jsn...), []byte("\n")...), nil } diff --git a/internal/record/v1.go b/internal/record/v1.go index 606988a..1620ccd 100644 --- a/internal/record/v1.go +++ b/internal/record/v1.go @@ -16,6 +16,7 @@ type V1 struct { ExitCode int `json:"exitCode"` // paths + // TODO: Do we need both pwd and real pwd? Home string `json:"home"` Pwd string `json:"pwd"` RealPwd string `json:"realPwd"` @@ -27,8 +28,9 @@ type V1 struct { // git info // origin is the most important GitOriginRemote string `json:"gitOriginRemote"` - // maybe branch could be useful - e.g. in monorepo ?? - GitBranch string `json:"gitBranch"` + // TODO: add GitBranch (v2 ?) + // maybe branch could be useful - e.g. in monorepo ?? + // GitBranch string `json:"gitBranch"` // what is this for ?? // session watching needs this @@ -36,11 +38,11 @@ type V1 struct { // records belong to sessions // PID int `json:"pid"` // needed for tracking of sessions but I think it shouldn't be part of V1 - SessionPID int `json:"sessionPID"` + // SessionPID int `json:"sessionPID"` // needed to because records are merged with parts with same "SessionID + Shlvl" // I don't think we need to save it - Shlvl int `json:"shlvl"` + // Shlvl int `json:"shlvl"` // time (before), duration of command Time float64 `json:"time"` diff --git a/internal/recordint/collect.go b/internal/recordint/collect.go index b1e62e7..645efe4 100644 --- a/internal/recordint/collect.go +++ b/internal/recordint/collect.go @@ -8,6 +8,7 @@ type Collect struct { Shlvl int // session watching SessionPID int + Shell string Rec record.V1 } @@ -18,4 +19,16 @@ type Postcollect struct { Shlvl int // session watching SessionPID int + + RecordID string + ExitCode int + Duration float64 +} + +type SessionInit struct { + // record merging + SessionID string + Shlvl int + // session watching + SessionPID int } diff --git a/internal/recordint/enriched.go b/internal/recordint/enriched.go deleted file mode 100644 index b2141a4..0000000 --- a/internal/recordint/enriched.go +++ /dev/null @@ -1,51 +0,0 @@ -package recordint - -import ( - "github.com/curusarn/resh/internal/record" - "github.com/curusarn/resh/internal/recutil" -) - -// TODO: This all seems excessive -// TODO: V1 should be converted directly to SearchApp record - -// EnrichedRecord - record enriched with additional data -type Enriched struct { - // TODO: think about if it really makes sense to have this based on V1 - record.V1 - - // TODO: drop some/all of this - // enriching fields - added "later" - Command string `json:"command"` - FirstWord string `json:"firstWord"` - Invalid bool `json:"invalid"` - SeqSessionID uint64 `json:"seqSessionId"` - LastRecordOfSession bool `json:"lastRecordOfSession"` - DebugThisRecord bool `json:"debugThisRecord"` - Errors []string `json:"errors"` - // SeqSessionID uint64 `json:"seqSessionId,omitempty"` -} - -// Enriched - returns enriched record -func NewEnrichedFromV1(r *record.V1) Enriched { - rec := Enriched{Record: r} - // normlize git remote - rec.GitOriginRemote = NormalizeGitRemote(rec.GitOriginRemote) - rec.GitOriginRemoteAfter = NormalizeGitRemote(rec.GitOriginRemoteAfter) - // Get command/first word from commandline - var err error - err = recutil.Validate(r) - if err != nil { - rec.Errors = append(rec.Errors, "Validate error:"+err.Error()) - // rec, _ := record.ToString() - // sugar.Println("Invalid command:", rec) - rec.Invalid = true - } - rec.Command, rec.FirstWord, err = GetCommandAndFirstWord(r.CmdLine) - if err != nil { - rec.Errors = append(rec.Errors, "GetCommandAndFirstWord error:"+err.Error()) - // rec, _ := record.ToString() - // sugar.Println("Invalid command:", rec) - rec.Invalid = true // should this be really invalid ? - } - return rec -} diff --git a/internal/recordint/searchapp.go b/internal/recordint/searchapp.go index 965ebce..a80b018 100644 --- a/internal/recordint/searchapp.go +++ b/internal/recordint/searchapp.go @@ -1,5 +1,12 @@ package recordint +import ( + "net/url" + "strings" + + giturls "github.com/whilp/git-urls" +) + // SearchApp record used for sending records to RESH-CLI type SearchApp struct { IsRaw bool @@ -14,6 +21,9 @@ type SearchApp struct { ExitCode int Time float64 + + // file index + Idx int } // NewCliRecordFromCmdLine @@ -25,16 +35,39 @@ func NewSearchAppFromCmdLine(cmdLine string) SearchApp { } // NewCliRecord from EnrichedRecord -func NewSearchApp(r *Enriched) SearchApp { +func NewSearchApp(r *Indexed) SearchApp { + // TODO: we used to validate records with recutil.Validate() return SearchApp{ - IsRaw: false, - SessionID: r.SessionID, - CmdLine: r.CmdLine, - Host: r.Hostname, - Pwd: r.Pwd, - Home: r.Home, - GitOriginRemote: r.GitOriginRemote, - ExitCode: r.ExitCode, - Time: r.Time, + IsRaw: false, + SessionID: r.Rec.SessionID, + CmdLine: r.Rec.CmdLine, + Host: r.Rec.Hostname, + Pwd: r.Rec.Pwd, + Home: r.Rec.Home, + // TODO: is this the right place to normalize the git remote + GitOriginRemote: normalizeGitRemote(r.Rec.GitOriginRemote), + ExitCode: r.Rec.ExitCode, + Time: r.Rec.Time, + + Idx: r.Idx, + } +} + +// TODO: maybe move this to a more appropriate place +// normalizeGitRemote helper +func normalizeGitRemote(gitRemote string) string { + if strings.HasSuffix(gitRemote, ".git") { + gitRemote = gitRemote[:len(gitRemote)-4] + } + parsedURL, err := giturls.Parse(gitRemote) + if err != nil { + // TODO: log this error + return gitRemote + } + if parsedURL.User == nil || parsedURL.User.Username() == "" { + parsedURL.User = url.User("git") } + // TODO: figure out what scheme we want + parsedURL.Scheme = "git+ssh" + return parsedURL.String() } diff --git a/internal/records/records.go b/internal/records/records.go index 1747547..a9972d2 100644 --- a/internal/records/records.go +++ b/internal/records/records.go @@ -2,269 +2,13 @@ package records import ( "bufio" - "encoding/json" - "fmt" - "io" "os" - "strconv" "strings" "github.com/curusarn/resh/internal/histlist" "go.uber.org/zap" ) -// BaseRecord - common base for Record and FallbackRecord -type BaseRecord struct { - // core - CmdLine string `json:"cmdLine"` - ExitCode int `json:"exitCode"` - Shell string `json:"shell"` - Uname string `json:"uname"` - SessionID string `json:"sessionId"` - RecordID string `json:"recordId"` - - // posix - Home string `json:"home"` - Lang string `json:"lang"` - LcAll string `json:"lcAll"` - Login string `json:"login"` - //Path string `json:"path"` - Pwd string `json:"pwd"` - PwdAfter string `json:"pwdAfter"` - ShellEnv string `json:"shellEnv"` - Term string `json:"term"` - - // non-posix"` - RealPwd string `json:"realPwd"` - RealPwdAfter string `json:"realPwdAfter"` - Pid int `json:"pid"` - SessionPID int `json:"sessionPid"` - Host string `json:"host"` - Hosttype string `json:"hosttype"` - Ostype string `json:"ostype"` - Machtype string `json:"machtype"` - Shlvl int `json:"shlvl"` - - // before after - TimezoneBefore string `json:"timezoneBefore"` - TimezoneAfter string `json:"timezoneAfter"` - - RealtimeBefore float64 `json:"realtimeBefore"` - RealtimeAfter float64 `json:"realtimeAfter"` - RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` - RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` - - RealtimeDuration float64 `json:"realtimeDuration"` - RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` - RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` - //Logs []string `json: "logs"` - - GitDir string `json:"gitDir"` - GitRealDir string `json:"gitRealDir"` - GitOriginRemote string `json:"gitOriginRemote"` - GitDirAfter string `json:"gitDirAfter"` - GitRealDirAfter string `json:"gitRealDirAfter"` - GitOriginRemoteAfter string `json:"gitOriginRemoteAfter"` - MachineID string `json:"machineId"` - - OsReleaseID string `json:"osReleaseId"` - OsReleaseVersionID string `json:"osReleaseVersionId"` - OsReleaseIDLike string `json:"osReleaseIdLike"` - OsReleaseName string `json:"osReleaseName"` - OsReleasePrettyName string `json:"osReleasePrettyName"` - - ReshUUID string `json:"reshUuid"` - ReshVersion string `json:"reshVersion"` - ReshRevision string `json:"reshRevision"` - - // records come in two parts (collect and postcollect) - PartOne bool `json:"partOne,omitempty"` // false => part two - PartsMerged bool `json:"partsMerged"` - // special flag -> not an actual record but an session end - SessionExit bool `json:"sessionExit,omitempty"` - - // recall metadata - Recalled bool `json:"recalled"` - RecallHistno int `json:"recallHistno,omitempty"` - RecallStrategy string `json:"recallStrategy,omitempty"` - RecallActionsRaw string `json:"recallActionsRaw,omitempty"` - RecallActions []string `json:"recallActions,omitempty"` - RecallLastCmdLine string `json:"recallLastCmdLine"` - - // recall command - RecallPrefix string `json:"recallPrefix,omitempty"` - - // added by sanitizatizer - Sanitized bool `json:"sanitized,omitempty"` - CmdLength int `json:"cmdLength,omitempty"` -} - -// Record representing single executed command with its metadata -type Record struct { - BaseRecord - - Cols string `json:"cols"` - Lines string `json:"lines"` -} - -// EnrichedRecord - record enriched with additional data -type EnrichedRecord struct { - Record - - // enriching fields - added "later" - Command string `json:"command"` - FirstWord string `json:"firstWord"` - Invalid bool `json:"invalid"` - SeqSessionID uint64 `json:"seqSessionId"` - LastRecordOfSession bool `json:"lastRecordOfSession"` - DebugThisRecord bool `json:"debugThisRecord"` - Errors []string `json:"errors"` - // SeqSessionID uint64 `json:"seqSessionId,omitempty"` -} - -// FallbackRecord when record is too old and can't be parsed into regular Record -type FallbackRecord struct { - BaseRecord - // older version of the record where cols and lines are int - - Cols int `json:"cols"` // notice the int type - Lines int `json:"lines"` // notice the int type -} - -// Convert from FallbackRecord to Record -func Convert(r *FallbackRecord) Record { - return Record{ - BaseRecord: r.BaseRecord, - // these two lines are the only reason we are doing this - Cols: strconv.Itoa(r.Cols), - Lines: strconv.Itoa(r.Lines), - } -} - -// ToString - returns record the json -func (r EnrichedRecord) ToString() (string, error) { - jsonRec, err := json.Marshal(r) - if err != nil { - return "marshalling error", err - } - return string(jsonRec), nil -} - -// LoadFromFile loads records from 'fname' file -func LoadFromFile(sugar *zap.SugaredLogger, fname string) []Record { - const allowedErrors = 3 - var encounteredErrors int - var recs []Record - file, err := os.Open(fname) - if err != nil { - sugar.Error("Failed to open resh history file - skipping reading resh history", zap.Error(err)) - return recs - } - defer file.Close() - - reader := bufio.NewReader(file) - var i int - for { - var line string - line, err = reader.ReadString('\n') - if err != nil { - break - } - i++ - record := Record{} - fallbackRecord := FallbackRecord{} - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - encounteredErrors++ - sugar.Error("Could not decode line in resh history file", - "lineContents", line, - "lineNumber", i, - zap.Error(err), - ) - if encounteredErrors > allowedErrors { - sugar.Fatal("Encountered too many errors during decoding - exiting", - "allowedErrors", allowedErrors, - ) - } - } - record = Convert(&fallbackRecord) - } - recs = append(recs, record) - } - if err != io.EOF { - sugar.Error("Error while loading file", zap.Error(err)) - } - sugar.Infow("Loaded resh history records", - "recordCount", len(recs), - ) - if encounteredErrors > 0 { - // fix errors in the history file - sugar.Warnw("Some history records could not be decoded - fixing resh history file by dropping them", - "corruptedRecords", encounteredErrors, - ) - fnameBak := fname + ".bak" - sugar.Infow("Backing up current corrupted history file", - "backupFilename", fnameBak, - ) - err := copyFile(fname, fnameBak) - if err != nil { - sugar.Errorw("Failed to create a backup history file - aborting fixing history file", - "backupFilename", fnameBak, - zap.Error(err), - ) - return recs - } - sugar.Info("Writing resh history file without errors ...") - err = writeHistory(fname, recs) - if err != nil { - sugar.Errorw("Failed write fixed history file - aborting fixing history file", - "filename", fname, - zap.Error(err), - ) - } - } - return recs -} - -func copyFile(source, dest string) error { - from, err := os.Open(source) - if err != nil { - return err - } - defer from.Close() - - // to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666) - to, err := os.Create(dest) - if err != nil { - return err - } - defer to.Close() - - _, err = io.Copy(to, from) - if err != nil { - return err - } - return nil -} - -func writeHistory(fname string, history []Record) error { - file, err := os.Create(fname) - if err != nil { - return err - } - defer file.Close() - for _, rec := range history { - jsn, err := json.Marshal(rec) - if err != nil { - return fmt.Errorf("failed to encode record: %w", err) - } - file.Write(append(jsn, []byte("\n")...)) - } - return nil -} - // LoadCmdLinesFromZshFile loads cmdlines from zsh history file func LoadCmdLinesFromZshFile(sugar *zap.SugaredLogger, fname string) histlist.Histlist { hl := histlist.New(sugar) diff --git a/internal/records/records_test.go b/internal/records/records_test.go deleted file mode 100644 index 9bcb69f..0000000 --- a/internal/records/records_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package records - -import ( - "bufio" - "encoding/json" - "log" - "os" - "testing" -) - -func GetTestRecords() []Record { - file, err := os.Open("testdata/resh_history.json") - if err != nil { - log.Fatalf("Failed to open resh history file: %v", err) - } - defer file.Close() - - var recs []Record - scanner := bufio.NewScanner(file) - for scanner.Scan() { - record := Record{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - log.Fatalf("Error decoding record: '%s'; err: %v", line, err) - } - recs = append(recs, record) - } - return recs -} - -func GetTestEnrichedRecords() []EnrichedRecord { - var recs []EnrichedRecord - for _, rec := range GetTestRecords() { - recs = append(recs, Enriched(rec)) - } - return recs -} - -func TestToString(t *testing.T) { - for _, rec := range GetTestEnrichedRecords() { - _, err := rec.ToString() - if err != nil { - t.Error("ToString() failed") - } - } -} - -func TestEnriched(t *testing.T) { - record := Record{BaseRecord: BaseRecord{CmdLine: "cmd arg1 arg2"}} - enriched := Enriched(record) - if enriched.FirstWord != "cmd" || enriched.Command != "cmd" { - t.Error("Enriched() returned reocord w/ wrong Command OR FirstWord") - } -} - -func TestValidate(t *testing.T) { - record := EnrichedRecord{} - if record.Validate() == nil { - t.Error("Validate() didn't return an error for invalid record") - } - record.CmdLine = "cmd arg" - record.FirstWord = "cmd" - record.Command = "cmd" - time := 1234.5678 - record.RealtimeBefore = time - record.RealtimeAfter = time - record.RealtimeBeforeLocal = time - record.RealtimeAfterLocal = time - pwd := "/pwd" - record.Pwd = pwd - record.PwdAfter = pwd - record.RealPwd = pwd - record.RealPwdAfter = pwd - if record.Validate() != nil { - t.Error("Validate() returned an error for a valid record") - } -} - -func TestGetCommandAndFirstWord(t *testing.T) { - cmd, stWord, err := GetCommandAndFirstWord("cmd arg1 arg2") - if err != nil || cmd != "cmd" || stWord != "cmd" { - t.Error("GetCommandAndFirstWord() returned wrong Command OR FirstWord") - } -} diff --git a/internal/records/testdata/resh_history.json b/internal/records/testdata/resh_history.json deleted file mode 100644 index 40f43ab..0000000 --- a/internal/records/testdata/resh_history.json +++ /dev/null @@ -1,27 +0,0 @@ -{"cmdLine":"ls","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"d5c0fe70-c80b-4715-87cb-f8d8d5b4c673","cols":"80","lines":"24","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14560,"sessionPid":14560,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1566762905.173595,"realtimeAfter":1566762905.1894295,"realtimeBeforeLocal":1566770105.173595,"realtimeAfterLocal":1566770105.1894295,"realtimeDuration":0.015834569931030273,"realtimeSinceSessionStart":1.7122540473937988,"realtimeSinceBoot":20766.542254047396,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"find . -name applications","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567420001.2531302,"realtimeAfter":1567420002.4311218,"realtimeBeforeLocal":1567427201.2531302,"realtimeAfterLocal":1567427202.4311218,"realtimeDuration":1.1779916286468506,"realtimeSinceSessionStart":957.4848053455353,"realtimeSinceBoot":2336.594805345535,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"desktop-file-validate curusarn.sync-clipboards.desktop ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/.local/share/applications","pwdAfter":"/home/simon/.local/share/applications","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/.local/share/applications","realPwdAfter":"/home/simon/.local/share/applications","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567421748.2965438,"realtimeAfter":1567421748.3068867,"realtimeBeforeLocal":1567428948.2965438,"realtimeAfterLocal":1567428948.3068867,"realtimeDuration":0.010342836380004883,"realtimeSinceSessionStart":2704.528218984604,"realtimeSinceBoot":4083.6382189846036,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"cat /tmp/extensions | grep '.'","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461416.6871984,"realtimeAfter":1567461416.7336714,"realtimeBeforeLocal":1567468616.6871984,"realtimeAfterLocal":1567468616.7336714,"realtimeDuration":0.046473026275634766,"realtimeSinceSessionStart":21.45597553253174,"realtimeSinceBoot":43752.03597553253,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"cd git/resh/","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461667.8806899,"realtimeAfter":1567461667.8949044,"realtimeBeforeLocal":1567468867.8806899,"realtimeAfterLocal":1567468867.8949044,"realtimeDuration":0.014214515686035156,"realtimeSinceSessionStart":272.64946699142456,"realtimeSinceBoot":44003.229466991426,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461707.6467602,"realtimeAfter":1567461707.7177293,"realtimeBeforeLocal":1567468907.6467602,"realtimeAfterLocal":1567468907.7177293,"realtimeDuration":0.0709691047668457,"realtimeSinceSessionStart":312.4155373573303,"realtimeSinceBoot":44042.99553735733,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"cat /tmp/extensions | grep '^\\.' | cut -f1 |tr '[:upper:]' '[:lower:]' ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461722.813049,"realtimeAfter":1567461722.8280325,"realtimeBeforeLocal":1567468922.813049,"realtimeAfterLocal":1567468922.8280325,"realtimeDuration":0.014983415603637695,"realtimeSinceSessionStart":327.581826210022,"realtimeSinceBoot":44058.161826210024,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"tig","exitCode":127,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461906.3896828,"realtimeAfter":1567461906.4084594,"realtimeBeforeLocal":1567469106.3896828,"realtimeAfterLocal":1567469106.4084594,"realtimeDuration":0.018776655197143555,"realtimeSinceSessionStart":511.1584599018097,"realtimeSinceBoot":44241.73845990181,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} -{"cmdLine":"resh-sanitize-history | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a3318c80-3521-4b22-aa64-ea0f6c641410","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14601,"sessionPid":14601,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567547116.2430356,"realtimeAfter":1567547116.7547352,"realtimeBeforeLocal":1567554316.2430356,"realtimeAfterLocal":1567554316.7547352,"realtimeDuration":0.5116996765136719,"realtimeSinceSessionStart":15.841878414154053,"realtimeSinceBoot":30527.201878414155,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo pacman -S ansible","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609042.0166302,"realtimeAfter":1567609076.9726007,"realtimeBeforeLocal":1567616242.0166302,"realtimeAfterLocal":1567616276.9726007,"realtimeDuration":34.95597052574158,"realtimeSinceSessionStart":1617.0794131755829,"realtimeSinceBoot":6120.029413175583,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"vagrant up","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609090.7359188,"realtimeAfter":1567609098.3125577,"realtimeBeforeLocal":1567616290.7359188,"realtimeAfterLocal":1567616298.3125577,"realtimeDuration":7.57663893699646,"realtimeSinceSessionStart":1665.798701763153,"realtimeSinceBoot":6168.748701763153,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo modprobe vboxnetflt","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609143.2847652,"realtimeAfter":1567609143.3116078,"realtimeBeforeLocal":1567616343.2847652,"realtimeAfterLocal":1567616343.3116078,"realtimeDuration":0.026842594146728516,"realtimeSinceSessionStart":1718.3475482463837,"realtimeSinceBoot":6221.2975482463835,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"echo $RANDOM","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"8ddacadc-6e73-483c-b347-4e18df204466","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":31387,"sessionPid":31387,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567727039.6540458,"realtimeAfter":1567727039.6629689,"realtimeBeforeLocal":1567734239.6540458,"realtimeAfterLocal":1567734239.6629689,"realtimeDuration":0.008923053741455078,"realtimeSinceSessionStart":1470.7667458057404,"realtimeSinceBoot":18495.01674580574,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"make resh-evaluate ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567977478.9672194,"realtimeAfter":1567977479.5449634,"realtimeBeforeLocal":1567984678.9672194,"realtimeAfterLocal":1567984679.5449634,"realtimeDuration":0.5777440071105957,"realtimeSinceSessionStart":5738.577540636063,"realtimeSinceBoot":20980.42754063606,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"cat ~/.resh_history.json | grep \"./resh-eval\" | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"105","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567986105.3988302,"realtimeAfter":1567986105.4809113,"realtimeBeforeLocal":1567993305.3988302,"realtimeAfterLocal":1567993305.4809113,"realtimeDuration":0.08208107948303223,"realtimeSinceSessionStart":14365.00915145874,"realtimeSinceBoot":29606.85915145874,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git c \"add sanitized flag to record, add Enrich() to record\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063976.9103937,"realtimeAfter":1568063976.9326868,"realtimeBeforeLocal":1568071176.9103937,"realtimeAfterLocal":1568071176.9326868,"realtimeDuration":0.0222930908203125,"realtimeSinceSessionStart":92236.52071499825,"realtimeSinceBoot":107478.37071499825,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063978.2340608,"realtimeAfter":1568063978.252463,"realtimeBeforeLocal":1568071178.2340608,"realtimeAfterLocal":1568071178.252463,"realtimeDuration":0.0184023380279541,"realtimeSinceSessionStart":92237.84438204765,"realtimeSinceBoot":107479.69438204766,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git a evaluate/results.go ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063989.0446353,"realtimeAfter":1568063989.2452207,"realtimeBeforeLocal":1568071189.0446353,"realtimeAfterLocal":1568071189.2452207,"realtimeDuration":0.20058536529541016,"realtimeSinceSessionStart":92248.65495657921,"realtimeSinceBoot":107490.50495657921,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo pacman -S python-pip","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072068.3557143,"realtimeAfter":1568072070.7509863,"realtimeBeforeLocal":1568079268.3557143,"realtimeAfterLocal":1568079270.7509863,"realtimeDuration":2.3952720165252686,"realtimeSinceSessionStart":100327.96603560448,"realtimeSinceBoot":115569.81603560448,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"pip3 install matplotlib","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072088.5575967,"realtimeAfter":1568072094.372314,"realtimeBeforeLocal":1568079288.5575967,"realtimeAfterLocal":1568079294.372314,"realtimeDuration":5.8147172927856445,"realtimeSinceSessionStart":100348.16791796684,"realtimeSinceBoot":115590.01791796685,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"sudo pip3 install matplotlib","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072106.138616,"realtimeAfter":1568072115.1124601,"realtimeBeforeLocal":1568079306.138616,"realtimeAfterLocal":1568079315.1124601,"realtimeDuration":8.973844051361084,"realtimeSinceSessionStart":100365.7489373684,"realtimeSinceBoot":115607.5989373684,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"./resh-evaluate --plotting-script evaluate/resh-evaluate-plot.py --input ~/git/resh_private/history_data/simon/dell/resh_history.json ","exitCode":130,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568076266.9364285,"realtimeAfter":1568076288.1131275,"realtimeBeforeLocal":1568083466.9364285,"realtimeAfterLocal":1568083488.1131275,"realtimeDuration":21.176698923110962,"realtimeSinceSessionStart":104526.54674983025,"realtimeSinceBoot":119768.39674983025,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} -{"cmdLine":"git c \"Add a bunch of useless comments to make linter happy\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"04050353-a97d-4435-9248-f47dd08b2f2a","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":14702,"sessionPid":14702,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569456045.8763022,"realtimeAfter":1569456045.9030173,"realtimeBeforeLocal":1569463245.8763022,"realtimeAfterLocal":1569463245.9030173,"realtimeDuration":0.02671504020690918,"realtimeSinceSessionStart":2289.789242744446,"realtimeSinceBoot":143217.91924274445,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"fuck","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a4aadf03-610d-4731-ba94-5b7ce21e7bb9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":3413,"sessionPid":3413,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569687682.4250975,"realtimeAfter":1569687682.5877323,"realtimeBeforeLocal":1569694882.4250975,"realtimeAfterLocal":1569694882.5877323,"realtimeDuration":0.16263484954833984,"realtimeSinceSessionStart":264603.49496507645,"realtimeSinceBoot":374854.48496507644,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"code .","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709366.523767,"realtimeAfter":1569709367.516908,"realtimeBeforeLocal":1569716566.523767,"realtimeAfterLocal":1569716567.516908,"realtimeDuration":0.9931409358978271,"realtimeSinceSessionStart":23846.908839941025,"realtimeSinceBoot":396539.888839941,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"make test","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709371.89966,"realtimeAfter":1569709377.430194,"realtimeBeforeLocal":1569716571.89966,"realtimeAfterLocal":1569716577.430194,"realtimeDuration":5.530533790588379,"realtimeSinceSessionStart":23852.284733057022,"realtimeSinceBoot":396545.264733057,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} -{"cmdLine":"mkdir ~/git/resh/testdata","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"71529b60-2e7b-4d5b-8dc1-6d0740b58e9e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":21224,"sessionPid":21224,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709838.4642656,"realtimeAfter":1569709838.4718792,"realtimeBeforeLocal":1569717038.4642656,"realtimeAfterLocal":1569717038.4718792,"realtimeDuration":0.007613658905029297,"realtimeSinceSessionStart":9.437154054641724,"realtimeSinceBoot":397011.02715405467,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} diff --git a/internal/recutil/recutil.go b/internal/recutil/recutil.go index 96d8baa..e2b3c85 100644 --- a/internal/recutil/recutil.go +++ b/internal/recutil/recutil.go @@ -2,93 +2,50 @@ package recutil import ( "errors" - "net/url" - "strings" "github.com/curusarn/resh/internal/record" - "github.com/mattn/go-shellwords" - giturls "github.com/whilp/git-urls" + "github.com/curusarn/resh/internal/recordint" ) -// NormalizeGitRemote helper -func NormalizeGitRemote(gitRemote string) string { - if strings.HasSuffix(gitRemote, ".git") { - gitRemote = gitRemote[:len(gitRemote)-4] - } - parsedURL, err := giturls.Parse(gitRemote) - if err != nil { - // TODO: log this error - return gitRemote - } - if parsedURL.User == nil || parsedURL.User.Username() == "" { - parsedURL.User = url.User("git") - } - // TODO: figure out what scheme we want - parsedURL.Scheme = "git+ssh" - return parsedURL.String() -} - +// TODO: reintroduce validation // Validate returns error if the record is invalid -func Validate(r *record.V1) error { - if r.CmdLine == "" { - return errors.New("There is no CmdLine") - } - if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 { - return errors.New("There is no Time") - } - if r.RealtimeBeforeLocal == 0 || r.RealtimeAfterLocal == 0 { - return errors.New("There is no Local Time") - } - if r.RealPwd == "" || r.RealPwdAfter == "" { - return errors.New("There is no Real Pwd") - } - if r.Pwd == "" || r.PwdAfter == "" { - return errors.New("There is no Pwd") - } - return nil -} +// func Validate(r *record.V1) error { +// if r.CmdLine == "" { +// return errors.New("There is no CmdLine") +// } +// if r.Time == 0 { +// return errors.New("There is no Time") +// } +// if r.RealPwd == "" { +// return errors.New("There is no Real Pwd") +// } +// if r.Pwd == "" { +// return errors.New("There is no Pwd") +// } +// return nil +// } +// TODO: maybe more to a more appropriate place +// TODO: cleanup the interface - stop modifying the part1 and returning a ew record at the same time // Merge two records (part1 - collect + part2 - postcollect) -func Merge(r1 *record.V1, r2 *record.V1) error { - if r1.PartOne == false || r2.PartOne { - return errors.New("Expected part1 and part2 of the same record - usage: Merge(part1, part2)") - } +func Merge(r1 *recordint.Collect, r2 *recordint.Collect) (record.V1, error) { if r1.SessionID != r2.SessionID { - return errors.New("Records to merge are not from the same sesion - r1:" + r1.SessionID + " r2:" + r2.SessionID) + return record.V1{}, errors.New("Records to merge are not from the same sesion - r1:" + r1.SessionID + " r2:" + r2.SessionID) } - if r1.CmdLine != r2.CmdLine { - return errors.New("Records to merge are not parts of the same records - r1:" + r1.CmdLine + " r2:" + r2.CmdLine) + if r1.Rec.RecordID != r2.Rec.RecordID { + return record.V1{}, errors.New("Records to merge do not have the same ID - r1:" + r1.Rec.RecordID + " r2:" + r2.Rec.RecordID) } - if r1.RecordID != r2.RecordID { - return errors.New("Records to merge do not have the same ID - r1:" + r1.RecordID + " r2:" + r2.RecordID) - } - r1.ExitCode = r2.ExitCode - r1.Duration = r2.Duration - r1.PartsMerged = true - r1.PartOne = false - return nil -} + r := recordint.Collect{ + SessionID: r1.SessionID, + Shlvl: r1.Shlvl, + SessionPID: r1.SessionPID, -// GetCommandAndFirstWord func -func GetCommandAndFirstWord(cmdLine string) (string, string, error) { - args, err := shellwords.Parse(cmdLine) - if err != nil { - // Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") - return "", "", err - } - if len(args) == 0 { - return "", "", nil - } - i := 0 - for true { - // commands in shell sometimes look like this `variable=something command argument otherArgument --option` - // to get the command we skip over tokens that contain '=' - if strings.ContainsRune(args[i], '=') && len(args) > i+1 { - i++ - continue - } - return args[i], args[0], nil + Rec: r1.Rec, } - return "ERROR", "ERROR", errors.New("failed to retrieve first word of command") + r.Rec.ExitCode = r2.Rec.ExitCode + r.Rec.Duration = r2.Rec.Duration + r.Rec.PartOne = false + r.Rec.PartsNotMerged = false + return r.Rec, nil } diff --git a/internal/searchapp/item.go b/internal/searchapp/item.go index c015428..33fc827 100644 --- a/internal/searchapp/item.go +++ b/internal/searchapp/item.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "golang.org/x/exp/utf8string" ) @@ -18,7 +18,7 @@ const dots = "…" type Item struct { isRaw bool - realtimeBefore float64 + time float64 // [host:]pwd differentHost bool @@ -105,8 +105,8 @@ func (i Item) DrawStatusLine(compactRendering bool, printedLineLength, realLineL if i.isRaw { return splitStatusLineToLines(i.CmdLine, printedLineLength, realLineLength) } - secs := int64(i.realtimeBefore) - nsecs := int64((i.realtimeBefore - float64(secs)) * 1e9) + secs := int64(i.time) + nsecs := int64((i.time - float64(secs)) * 1e9) tm := time.Unix(secs, nsecs) const timeFormat = "2006-01-02 15:04:05" timeString := tm.Format(timeFormat) @@ -142,8 +142,8 @@ func (i Item) DrawItemColumns(compactRendering bool, debug bool) ItemColumns { // DISPLAY // DISPLAY > date - secs := int64(i.realtimeBefore) - nsecs := int64((i.realtimeBefore - float64(secs)) * 1e9) + secs := int64(i.time) + nsecs := int64((i.time - float64(secs)) * 1e9) tm := time.Unix(secs, nsecs) var date string @@ -314,7 +314,7 @@ func properMatch(str, term, padChar string) bool { // NewItemFromRecordForQuery creates new item from record based on given query // returns error if the query doesn't match the record -func NewItemFromRecordForQuery(record records.CliRecord, query Query, debug bool) (Item, error) { +func NewItemFromRecordForQuery(record recordint.SearchApp, query Query, debug bool) (Item, error) { // Use numbers that won't add up to same score for any number of query words // query score weigth 1.51 const hitScore = 1.517 // 1 * 1.51 @@ -411,10 +411,10 @@ func NewItemFromRecordForQuery(record records.CliRecord, query Query, debug bool // if score <= 0 && !anyHit { // return Item{}, errors.New("no match for given record and query") // } - score += record.RealtimeBefore * timeScoreCoef + score += record.Time * timeScoreCoef it := Item{ - realtimeBefore: record.RealtimeBefore, + time: record.Time, differentHost: differentHost, host: record.Host, @@ -470,7 +470,7 @@ type RawItem struct { // NewRawItemFromRecordForQuery creates new item from record based on given query // returns error if the query doesn't match the record -func NewRawItemFromRecordForQuery(record records.CliRecord, terms []string, debug bool) (RawItem, error) { +func NewRawItemFromRecordForQuery(record recordint.SearchApp, terms []string, debug bool) (RawItem, error) { const hitScore = 1.0 const hitScoreConsecutive = 0.01 const properMatchScore = 0.3 @@ -489,7 +489,7 @@ func NewRawItemFromRecordForQuery(record records.CliRecord, terms []string, debu cmd = strings.ReplaceAll(cmd, term, highlightMatch(term)) } } - score += record.RealtimeBefore * timeScoreCoef + score += record.Time * timeScoreCoef // KEY for deduplication key := record.CmdLine diff --git a/internal/searchapp/test.go b/internal/searchapp/test.go index 36d55e4..9a2d284 100644 --- a/internal/searchapp/test.go +++ b/internal/searchapp/test.go @@ -3,20 +3,24 @@ package searchapp import ( "github.com/curusarn/resh/internal/histcli" "github.com/curusarn/resh/internal/msg" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recio" "go.uber.org/zap" ) // LoadHistoryFromFile ... func LoadHistoryFromFile(sugar *zap.SugaredLogger, historyPath string, numLines int) msg.CliResponse { - recs := records.LoadFromFile(sugar, historyPath) + rio := recio.New(sugar) + recs, _, err := rio.ReadFile(historyPath) + if err != nil { + sugar.Panicf("failed to read hisotry file: %w", err) + } if numLines != 0 && numLines < len(recs) { recs = recs[:numLines] } cliRecords := histcli.New() for i := len(recs) - 1; i >= 0; i-- { rec := recs[i] - cliRecords.AddRecord(rec) + cliRecords.AddRecord(&rec) } - return msg.CliResponse{CliRecords: cliRecords.List} + return msg.CliResponse{Records: cliRecords.List} } diff --git a/internal/sesswatch/sesswatch.go b/internal/sesswatch/sesswatch.go index e5a55ec..b20bb61 100644 --- a/internal/sesswatch/sesswatch.go +++ b/internal/sesswatch/sesswatch.go @@ -4,7 +4,7 @@ import ( "sync" "time" - "github.com/curusarn/resh/internal/records" + "github.com/curusarn/resh/internal/recordint" "github.com/mitchellh/go-ps" "go.uber.org/zap" ) @@ -21,7 +21,7 @@ type sesswatch struct { // Go runs the session watcher - watches sessions and sends func Go(sugar *zap.SugaredLogger, - sessionsToWatch chan records.Record, sessionsToWatchRecords chan records.Record, + sessionsToWatch chan recordint.SessionInit, sessionsToWatchRecords chan recordint.Collect, sessionsToDrop []chan string, sleepSeconds uint) { sw := sesswatch{ @@ -33,17 +33,17 @@ func Go(sugar *zap.SugaredLogger, go sw.waiter(sessionsToWatch, sessionsToWatchRecords) } -func (s *sesswatch) waiter(sessionsToWatch chan records.Record, sessionsToWatchRecords chan records.Record) { +func (s *sesswatch) waiter(sessionsToWatch chan recordint.SessionInit, sessionsToWatchRecords chan recordint.Collect) { for { func() { select { - case record := <-sessionsToWatch: + case rec := <-sessionsToWatch: // normal way to start watching a session - id := record.SessionID - pid := record.SessionPID + id := rec.SessionID + pid := rec.SessionPID sugar := s.sugar.With( - "sessionID", record.SessionID, - "sessionPID", record.SessionPID, + "sessionID", rec.SessionID, + "sessionPID", rec.SessionPID, ) s.mutex.Lock() defer s.mutex.Unlock() @@ -52,13 +52,13 @@ func (s *sesswatch) waiter(sessionsToWatch chan records.Record, sessionsToWatchR s.watchedSessions[id] = true go s.watcher(sugar, id, pid) } - case record := <-sessionsToWatchRecords: + case rec := <-sessionsToWatchRecords: // additional safety - watch sessions that were never properly initialized - id := record.SessionID - pid := record.SessionPID + id := rec.SessionID + pid := rec.SessionPID sugar := s.sugar.With( - "sessionID", record.SessionID, - "sessionPID", record.SessionPID, + "sessionID", rec.SessionID, + "sessionPID", rec.SessionPID, ) s.mutex.Lock() defer s.mutex.Unlock() diff --git a/scripts/hooks.sh b/scripts/hooks.sh index a734b7b..20d8c9e 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -8,30 +8,21 @@ __resh_preexec() { # core __RESH_COLLECT=1 __RESH_CMDLINE="$1" # not local to preserve it for postcollect (useful as sanity check) - local fpath_last_run="$__RESH_XDG_CACHE_HOME/collect_last_run_out.txt" - __resh_collect --cmdLine "$__RESH_CMDLINE" \ - >| "$fpath_last_run" 2>&1 || echo "resh-collect ERROR: $(head -n 1 $fpath_last_run)" + __resh_collect --cmdLine "$__RESH_CMDLINE" } # used for collect and collect --recall __resh_collect() { # posix - local __RESH_COLS="$COLUMNS" - local __RESH_LANG="$LANG" - local __RESH_LC_ALL="$LC_ALL" - local __RESH_LINES="$LINES" local __RESH_PWD="$PWD" # non-posix local __RESH_SHLVL="$SHLVL" - local __RESH_GIT_CDUP; __RESH_GIT_CDUP="$(git rev-parse --show-cdup 2>/dev/null)" - local __RESH_GIT_CDUP_EXIT_CODE=$? local __RESH_GIT_REMOTE; __RESH_GIT_REMOTE="$(git remote get-url origin 2>/dev/null)" local __RESH_GIT_REMOTE_EXIT_CODE=$? local __RESH_PID="$$" # current pid # time - local __RESH_TZ_BEFORE; __RESH_TZ_BEFORE=$(date +%z) # __RESH_RT_BEFORE="$EPOCHREALTIME" __RESH_RT_BEFORE=$(__resh_get_epochrealtime) @@ -54,38 +45,16 @@ __resh_collect() { resh-collect -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ -shell "$__RESH_SHELL" \ - -uname "$__RESH_UNAME" \ - -sessionId "$__RESH_SESSION_ID" \ - -recordId "$__RESH_RECORD_ID" \ - -cols "$__RESH_COLS" \ + -sessionID "$__RESH_SESSION_ID" \ + -recordID "$__RESH_RECORD_ID" \ -home "$__RESH_HOME" \ - -lang "$__RESH_LANG" \ - -lcAll "$__RESH_LC_ALL" \ - -lines "$__RESH_LINES" \ - -login "$__RESH_LOGIN" \ + -logname "$__RESH_LOGIN" \ -pwd "$__RESH_PWD" \ - -shellEnv "$__RESH_SHELL_ENV" \ - -term "$__RESH_TERM" \ - -pid "$__RESH_PID" \ - -sessionPid "$__RESH_SESSION_PID" \ - -host "$__RESH_HOST" \ - -hosttype "$__RESH_HOSTTYPE" \ - -ostype "$__RESH_OSTYPE" \ - -machtype "$__RESH_MACHTYPE" \ + -sessionPID "$__RESH_SESSION_PID" \ + -hostname "$__RESH_HOST" \ -shlvl "$__RESH_SHLVL" \ - -gitCdup "$__RESH_GIT_CDUP" \ - -gitCdupExitCode "$__RESH_GIT_CDUP_EXIT_CODE" \ -gitRemote "$__RESH_GIT_REMOTE" \ - -gitRemoteExitCode "$__RESH_GIT_REMOTE_EXIT_CODE" \ - -realtimeBefore "$__RESH_RT_BEFORE" \ - -realtimeSession "$__RESH_RT_SESSION" \ - -realtimeSessSinceBoot "$__RESH_RT_SESS_SINCE_BOOT" \ - -timezoneBefore "$__RESH_TZ_BEFORE" \ - -osReleaseId "$__RESH_OS_RELEASE_ID" \ - -osReleaseVersionId "$__RESH_OS_RELEASE_VERSION_ID" \ - -osReleaseIdLike "$__RESH_OS_RELEASE_ID_LIKE" \ - -osReleaseName "$__RESH_OS_RELEASE_NAME" \ - -osReleasePrettyName "$__RESH_OS_RELEASE_PRETTY_NAME" \ + -time "$__RESH_RT_BEFORE" \ "$@" return $? fi @@ -95,20 +64,8 @@ __resh_collect() { __resh_precmd() { local __RESH_EXIT_CODE=$? local __RESH_RT_AFTER - local __RESH_TZ_AFTER - local __RESH_PWD_AFTER - local __RESH_GIT_CDUP_AFTER - local __RESH_GIT_CDUP_EXIT_CODE_AFTER - local __RESH_GIT_REMOTE_AFTER - local __RESH_GIT_REMOTE_EXIT_CODE_AFTER local __RESH_SHLVL="$SHLVL" __RESH_RT_AFTER=$(__resh_get_epochrealtime) - __RESH_TZ_AFTER=$(date +%z) - __RESH_PWD_AFTER="$PWD" - __RESH_GIT_CDUP_AFTER="$(git rev-parse --show-cdup 2>/dev/null)" - __RESH_GIT_CDUP_EXIT_CODE_AFTER=$? - __RESH_GIT_REMOTE_AFTER="$(git remote get-url origin 2>/dev/null)" - __RESH_GIT_REMOTE_EXIT_CODE_AFTER=$? if [ -n "${__RESH_COLLECT}" ]; then if [ "$__RESH_VERSION" != "$(resh-postcollect -version)" ]; then # shellcheck source=shellrc.sh @@ -126,24 +83,14 @@ __resh_precmd() { fi fi if [ "$__RESH_VERSION" = "$(resh-postcollect -version)" ] && [ "$__RESH_REVISION" = "$(resh-postcollect -revision)" ]; then - local fpath_last_run="$__RESH_XDG_CACHE_HOME/postcollect_last_run_out.txt" resh-postcollect -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ - -cmdLine "$__RESH_CMDLINE" \ - -realtimeBefore "$__RESH_RT_BEFORE" \ + -timeBefore "$__RESH_RT_BEFORE" \ -exitCode "$__RESH_EXIT_CODE" \ - -sessionId "$__RESH_SESSION_ID" \ - -recordId "$__RESH_RECORD_ID" \ - -shell "$__RESH_SHELL" \ + -sessionID "$__RESH_SESSION_ID" \ + -recordID "$__RESH_RECORD_ID" \ -shlvl "$__RESH_SHLVL" \ - -pwdAfter "$__RESH_PWD_AFTER" \ - -gitCdupAfter "$__RESH_GIT_CDUP_AFTER" \ - -gitCdupExitCodeAfter "$__RESH_GIT_CDUP_EXIT_CODE_AFTER" \ - -gitRemoteAfter "$__RESH_GIT_REMOTE_AFTER" \ - -gitRemoteExitCodeAfter "$__RESH_GIT_REMOTE_EXIT_CODE_AFTER" \ - -realtimeAfter "$__RESH_RT_AFTER" \ - -timezoneAfter "$__RESH_TZ_AFTER" \ - >| "$fpath_last_run" 2>&1 || echo "resh-postcollect ERROR: $(head -n 1 $fpath_last_run)" + -timeAfter "$__RESH_RT_AFTER" fi __resh_reset_variables fi diff --git a/scripts/install.sh b/scripts/install.sh index 8d5aba9..336c4cf 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -135,7 +135,7 @@ bin/resh-control completion zsh > ~/.resh/zsh_completion.d/_reshctl echo "Copying more files ..." cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid -cp -f bin/resh-{daemon,control,collect,postcollect,session-init,config} ~/.resh/bin/ +cp -f bin/resh-{daemon,cli,control,collect,postcollect,session-init,config} ~/.resh/bin/ echo "Creating/updating config file ..." ./bin/resh-config-setup @@ -179,7 +179,8 @@ else pkill -SIGTERM "resh-daemon" || true fi # daemon uses xdg path variables -__resh_set_xdg_home_paths +# FIXME: this does not exist anymore +#__resh_set_xdg_home_paths __resh_run_daemon diff --git a/scripts/reshctl.sh b/scripts/reshctl.sh index d043366..69ab09d 100644 --- a/scripts/reshctl.sh +++ b/scripts/reshctl.sh @@ -74,8 +74,6 @@ resh() { elif [ $status_code = 130 ]; then true else - local fpath_last_run="$__RESH_XDG_CACHE_HOME/cli_last_run_out.txt" - echo "$buffer" >| "$fpath_last_run" - echo "resh-cli failed - check '$fpath_last_run' and '~/.resh/cli.log'" + printf "%s" "$buffer" >&2 fi } \ No newline at end of file diff --git a/scripts/shellrc.sh b/scripts/shellrc.sh index 2bc84ee..9092ff6 100644 --- a/scripts/shellrc.sh +++ b/scripts/shellrc.sh @@ -55,7 +55,8 @@ export __RESH_VERSION=$(resh-collect -version) # shellcheck disable=2155 export __RESH_REVISION=$(resh-collect -revision) -__resh_set_xdg_home_paths +# FIXME: this does not exist anymore +# __resh_set_xdg_home_paths __resh_run_daemon diff --git a/scripts/util.sh b/scripts/util.sh index 7383e57..8d616a5 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -47,19 +47,21 @@ __resh_get_epochrealtime() { fi } +# FIXME: figure out if stdout/stderr should be discarded __resh_run_daemon() { if [ -n "${ZSH_VERSION-}" ]; then setopt LOCAL_OPTIONS NO_NOTIFY NO_MONITOR fi - local fpath_last_run="$__RESH_XDG_CACHE_HOME/daemon_last_run_out.txt" if [ "$(uname)" = Darwin ]; then # hotfix - gnohup resh-daemon >| "$fpath_last_run" 2>&1 & disown + # gnohup resh-daemon 2>&1 & disown + gnohup resh-daemon & disown else # TODO: switch to nohup for consistency once you confirm that daemon is # not getting killed anymore on macOS - # nohup resh-daemon >| "$fpath_last_run" 2>&1 & disown - setsid resh-daemon >| "$fpath_last_run" 2>&1 & disown + nohup resh-daemon & disown + #nohup resh-daemon 2>&1 & disown + #setsid resh-daemon 2>&1 & disown fi } @@ -71,7 +73,7 @@ __resh_bash_completion_init() { . ~/.resh/bash_completion.d/_reshctl } -// TODO: redo this +# TODO: redo this __resh_zsh_completion_init() { # NOTE: this is hacky - each completion needs to be added individually # TODO: fix later @@ -131,32 +133,8 @@ __resh_session_init() { if [ "$__RESH_VERSION" = "$(resh-session-init -version)" ] && [ "$__RESH_REVISION" = "$(resh-session-init -revision)" ]; then resh-session-init -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ - -shell "$__RESH_SHELL" \ - -uname "$__RESH_UNAME" \ -sessionId "$__RESH_SESSION_ID" \ - -cols "$__RESH_COLS" \ - -home "$__RESH_HOME" \ - -lang "$__RESH_LANG" \ - -lcAll "$__RESH_LC_ALL" \ - -lines "$__RESH_LINES" \ - -login "$__RESH_LOGIN" \ - -shellEnv "$__RESH_SHELL_ENV" \ - -term "$__RESH_TERM" \ - -pid "$__RESH_PID" \ - -sessionPid "$__RESH_SESSION_PID" \ - -host "$__RESH_HOST" \ - -hosttype "$__RESH_HOSTTYPE" \ - -ostype "$__RESH_OSTYPE" \ - -machtype "$__RESH_MACHTYPE" \ - -shlvl "$__RESH_SHLVL" \ - -realtimeBefore "$__RESH_RT_BEFORE" \ - -realtimeSession "$__RESH_RT_SESSION" \ - -realtimeSessSinceBoot "$__RESH_RT_SESS_SINCE_BOOT" \ - -timezoneBefore "$__RESH_TZ_BEFORE" \ - -osReleaseId "$__RESH_OS_RELEASE_ID" \ - -osReleaseVersionId "$__RESH_OS_RELEASE_VERSION_ID" \ - -osReleaseIdLike "$__RESH_OS_RELEASE_ID_LIKE" \ - -osReleaseName "$__RESH_OS_RELEASE_NAME" \ - -osReleasePrettyName "$__RESH_OS_RELEASE_PRETTY_NAME" - fi + -sessionPid "$__RESH_SESSION_PID" + fi + } diff --git a/scripts/widgets.sh b/scripts/widgets.sh index fd0a0b9..e8acd55 100644 --- a/scripts/widgets.sh +++ b/scripts/widgets.sh @@ -16,8 +16,6 @@ __resh_widget_control_R() { local git_remote; git_remote="$(git remote get-url origin 2>/dev/null)" BUFFER=$(resh-cli --sessionID "$__RESH_SESSION_ID" --host "$__RESH_HOST" --pwd "$PWD" --gitOriginRemote "$git_remote" --query "$BUFFER") status_code=$? - local fpath_last_run="$__RESH_XDG_CACHE_HOME/cli_last_run_out.txt" - touch "$fpath_last_run" if [ $status_code = 111 ]; then # execute if [ -n "${ZSH_VERSION-}" ]; then @@ -35,8 +33,8 @@ __resh_widget_control_R() { bind -x '"\u[32~": __resh_nop' fi else - echo "$BUFFER" >| "$fpath_last_run" - echo "# RESH SEARCH APP failed - sorry for the inconvinience - check '$fpath_last_run' and '~/.resh/cli.log'" + echo "RESH SEARCH APP failed" + printf "%s" "$buffer" >&2 BUFFER="$PREVBUFFER" fi CURSOR=${#BUFFER} From 49ac6197d6fdb41eaea265195df120ba24bc72a0 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 19 Oct 2022 13:41:05 +0200 Subject: [PATCH 22/35] various improvements and changes --- cmd/collect/main.go | 10 +++--- cmd/config/main.go | 4 +-- cmd/daemon/run-server.go | 2 +- cmd/install-utils/backup.go | 9 +++++ cmd/install-utils/main.go | 35 +++++++++--------- cmd/install-utils/migrate.go | 10 ++++-- cmd/postcollect/main.go | 2 +- internal/cfg/cfg.go | 64 ++++++++++++++++++++++++++++----- internal/cfg/migrate.go | 7 ++++ internal/device/device.go | 49 +++++++++++++++++++++++++ internal/deviceid/deviceid.go | 18 ---------- internal/output/output.go | 4 ++- internal/recconv/recconv.go | 18 +++++----- internal/recload/recload.go | 1 - internal/record/v1.go | 22 +++++++----- internal/recordint/searchapp.go | 7 ++-- scripts/hooks.sh | 10 +++--- scripts/install.sh | 30 ++++++++-------- scripts/shellrc.sh | 38 +++++--------------- scripts/util.sh | 56 ++--------------------------- 20 files changed, 212 insertions(+), 184 deletions(-) create mode 100644 internal/device/device.go delete mode 100644 internal/deviceid/deviceid.go delete mode 100644 internal/recload/recload.go diff --git a/cmd/collect/main.go b/cmd/collect/main.go index 7f19e0a..1974870 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -54,8 +54,8 @@ func main() { shell := flag.String("shell", "", "current shell") - logname := flag.String("logname", "", "$LOGNAME") - hostname := flag.String("hostname", "", "$HOSTNAME") + // logname := flag.String("logname", "", "$LOGNAME") + device := flag.String("device", "", "device name, usually $HOSTNAME") // non-posix shlvl := flag.Int("shlvl", -1, "$SHLVL") @@ -111,12 +111,12 @@ func main() { Pwd: *pwd, RealPwd: realPwd, - Logname: *logname, - Hostname: *hostname, + // Logname: *logname, + Device: *device, GitOriginRemote: *gitRemote, - Time: time, + Time: fmt.Sprintf("%.4f", time), PartOne: true, PartsNotMerged: true, diff --git a/cmd/config/main.go b/cmd/config/main.go index e908817..eaa68b8 100644 --- a/cmd/config/main.go +++ b/cmd/config/main.go @@ -39,9 +39,9 @@ func main() { case "port": fmt.Println(config.Port) case "sesswatchperiodseconds": - fmt.Println(config.SesswatchPeriodSeconds) + fmt.Println(config.SessionWatchPeriodSeconds) case "sesshistinithistorysize": - fmt.Println(config.SesshistInitHistorySize) + fmt.Println(config.ReshHistoryMinSize) default: fmt.Println("Error: illegal --key!") os.Exit(1) diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index 00d9ede..afe77a1 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -57,7 +57,7 @@ func (s *Server) Run() { sesswatchSessionsToWatch, sesswatchRecords, sessionDropSubscribers, - s.config.SesswatchPeriodSeconds, + s.config.SessionWatchPeriodSeconds, ) // handlers diff --git a/cmd/install-utils/backup.go b/cmd/install-utils/backup.go index 9148164..1121535 100644 --- a/cmd/install-utils/backup.go +++ b/cmd/install-utils/backup.go @@ -1,5 +1,14 @@ package main func backup() { + panic("Backup not implemented yet!") + // Backup ~/.resh + // Backup xdg_data/resh/history.reshjson + // TODO: figure out history file localtions when using history sync +} +func rollback() { + panic("Rollback not implemented yet!") + // Rollback ~/.resh + // Rollback history } diff --git a/cmd/install-utils/main.go b/cmd/install-utils/main.go index 6c7209e..0d09a30 100644 --- a/cmd/install-utils/main.go +++ b/cmd/install-utils/main.go @@ -1,7 +1,6 @@ package main import ( - "flag" "fmt" "os" ) @@ -12,17 +11,16 @@ var commit string var developement bool func main() { - var command string - flag.StringVar(&command, "command", "", "Utility to run") - flag.Parse() - + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "ERROR: Not eonugh arguments\n") + printUsage(os.Stderr) + } + command := os.Args[1] switch command { case "backup": backup() case "rollback": - // FIXME - panic("Rollback not implemented yet!") - // rollback() + rollback() case "migrate-config": migrateConfig() case "migrate-history": @@ -30,22 +28,23 @@ func main() { case "help": printUsage(os.Stdout) default: - fmt.Fprintf(os.Stderr, "ERROR: Unknown command") + fmt.Fprintf(os.Stderr, "ERROR: Unknown command: %s\n", command) printUsage(os.Stderr) } } func printUsage(f *os.File) { usage := ` - Utils used during resh instalation +USAGE: ./install-utils COMMAND +Utils used during RESH instalation. + +COMMANDS: + backup backup resh installation and data + rollback restore resh installation and data from backup + migrate-config update config to reflect updates + migrate-history update history to reflect updates + help show this help - USAGE: ./install-utils COMMAND - COMMANDS: - backup backup resh installation and data - rollback restore resh installation and data from backup - migrate-config update config to reflect updates - migrate-history update history to reflect updates - help show this help - ` +` fmt.Fprintf(f, usage) } diff --git a/cmd/install-utils/migrate.go b/cmd/install-utils/migrate.go index e0bd34e..e68ffe2 100644 --- a/cmd/install-utils/migrate.go +++ b/cmd/install-utils/migrate.go @@ -19,7 +19,7 @@ func migrateConfig() { os.Exit(1) } if changes { - fmt.Printf("Config file format has changed since last update - your config was updated to reflect the changes.\n") + fmt.Printf("RESH config file format has changed since last update - your config was updated to reflect the changes.\n") } } @@ -30,7 +30,11 @@ func migrateHistory() { // } // TODO: Find history in: + // - .resh/history.json (copy) - message user to delete the file once they confirm the new setup works + // - .resh_history.json (copy) - message user to delete the file once they confirm the new setup works // - xdg_data/resh/history.reshjson - // - .resh_history.json - // - .resh/history.json + + // Read xdg_data/resh/history.reshjson + // Write xdg_data/resh/history.reshjson + // the old format can be found in the backup dir } diff --git a/cmd/postcollect/main.go b/cmd/postcollect/main.go index 41a3172..2023a6a 100644 --- a/cmd/postcollect/main.go +++ b/cmd/postcollect/main.go @@ -84,7 +84,7 @@ func main() { SessionID: *sessionID, ExitCode: *exitCode, - Duration: duration, + Duration: fmt.Sprintf("%.4f", duration), PartsNotMerged: true, }, diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 4df3d94..76bd73b 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -44,12 +44,16 @@ type Config struct { // Debug mode for search app Debug bool - // SesswatchPeriodSeconds is how often should daemon check if terminal + // SessionWatchPeriodSeconds is how often should daemon check if terminal // sessions are still alive - SesswatchPeriodSeconds uint - // SesshistInitHistorySize is how large resh history needs to be for + // There is not much need to adjust the value both memory overhead of watched sessions + // and the CPU overhead of chacking them are relatively low + SessionWatchPeriodSeconds uint + // ReshHistoryMinSize is how large resh history needs to be for // daemon to ignore standard shell history files - SesshistInitHistorySize int + // Ignoring standard shell history gives us more consistent experience + // but you can increase this to something large to see standard shell history in RESH search + ReshHistoryMinSize int } // defaults for config @@ -58,11 +62,53 @@ var defaults = Config{ LogLevel: zap.InfoLevel, BindControlR: true, - Debug: false, - SesswatchPeriodSeconds: 600, - SesshistInitHistorySize: 1000, + Debug: false, + SessionWatchPeriodSeconds: 600, + ReshHistoryMinSize: 1000, } +const headerComment = `## +###################### +## RESH config (v1) ## +###################### +## Here you can find info about RESH configuration options. +## You can uncomment the options and custimize them. + +## Required. +## The config format can change in future versions. +## ConfigVersion helps us seemlessly upgrade to the new formats. +# ConfigVersion = "v1" + +## Port used by RESH daemon and rest of the components to communicate. +## Make sure to restart the daemon (pkill resh-daemon) when you change it. +# Port = 2627 + +## Controls how much and how detailed logs all RESH components produce. +## Use "debug" for full logs when you encounter an issue +## Options: "debug", "info", "warn", "error", "fatal" +# LogLevel = "info" + +## When BindControlR is "true" RESH search app is bound to CTRL+R on terminal startuA +# BindControlR = true + +## When Debug is "true" the RESH search app runs in debug mode. +## This is useful for development. +# Debug = false + +## Daemon keeps track of running terminal sessions. +## SessionWatchPeriodSeconds controls how often daemon checks if the sessions are still alive. +## You shouldn't need to adjust this. +# SessionWatchPeriodSeconds = 600 + +## When RESH is first installed there is no RESH history so there is nothing to search. +## As a temporary woraround, RESH daemon parses bash/zsh shell history and searches it. +## Once RESH history is big enough RESH stops using bash/zsh history. +## ReshHistoryMinSize controls how big RESH history needs to be before this happens. +## You can increase this this to e.g. 10000 to get RESH to use bash/zsh history longer. +# ReshHistoryMinSize = 1000 + +` + func getConfigPath() (string, error) { fname := "resh.toml" xdgDir, found := os.LookupEnv("XDG_CONFIG_HOME") @@ -100,10 +146,10 @@ func processAndFillDefaults(configF *configFile) (Config, error) { config.Port = *configF.Port } if configF.SesswatchPeriodSeconds != nil { - config.SesswatchPeriodSeconds = *configF.SesswatchPeriodSeconds + config.SessionWatchPeriodSeconds = *configF.SesswatchPeriodSeconds } if configF.SesshistInitHistorySize != nil { - config.SesshistInitHistorySize = *configF.SesshistInitHistorySize + config.ReshHistoryMinSize = *configF.SesshistInitHistorySize } var err error diff --git a/internal/cfg/migrate.go b/internal/cfg/migrate.go index 4f91278..fbbb214 100644 --- a/internal/cfg/migrate.go +++ b/internal/cfg/migrate.go @@ -57,12 +57,19 @@ func Migrate() (bool, error) { return true, nil } +// writeConfig should only be used when migrating config to new version +// writing the config file discards all comments in the config file (limitation of TOML library) +// to make up for lost comments we add header comment to the start of the file func writeConfig(config *configFile, path string) error { file, err := os.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0666) if err != nil { return fmt.Errorf("could not open config for writing: %w", err) } defer file.Close() + _, err = file.WriteString(headerComment) + if err != nil { + return fmt.Errorf("could not write config header: %w", err) + } err = toml.NewEncoder(file).Encode(config) if err != nil { return fmt.Errorf("could not encode config: %w", err) diff --git a/internal/device/device.go b/internal/device/device.go new file mode 100644 index 0000000..8a34ead --- /dev/null +++ b/internal/device/device.go @@ -0,0 +1,49 @@ +package device + +import ( + "fmt" + "os" + "path" + "strings" +) + +func GetID(dataDir string) (string, error) { + fname := "device-id" + dat, err := os.ReadFile(path.Join(dataDir, fname)) + if err != nil { + return "", fmt.Errorf("could not read file with device-id: %w", err) + } + id := strings.TrimRight(string(dat), "\n") + return id, nil +} + +func GetName(dataDir string) (string, error) { + fname := "device-name" + dat, err := os.ReadFile(path.Join(dataDir, fname)) + if err != nil { + return "", fmt.Errorf("could not read file with device-name: %w", err) + } + name := strings.TrimRight(string(dat), "\n") + return name, nil +} + +// TODO: implement, possibly with a better name +// func CheckID(dataDir string) (string, error) { +// fname := "device-id" +// dat, err := os.ReadFile(path.Join(dataDir, fname)) +// if err != nil { +// return "", fmt.Errorf("could not read file with device-id: %w", err) +// } +// id := strings.TrimRight(string(dat), "\n") +// return id, nil +// } +// +// func CheckName(dataDir string) (string, error) { +// fname := "device-id" +// dat, err := os.ReadFile(path.Join(dataDir, fname)) +// if err != nil { +// return "", fmt.Errorf("could not read file with device-id: %w", err) +// } +// id := strings.TrimRight(string(dat), "\n") +// return id, nil +// } diff --git a/internal/deviceid/deviceid.go b/internal/deviceid/deviceid.go deleted file mode 100644 index 79a080d..0000000 --- a/internal/deviceid/deviceid.go +++ /dev/null @@ -1,18 +0,0 @@ -package deviceid - -import ( - "fmt" - "os" - "path" - "strings" -) - -func Get(dataDir string) (string, error) { - fname := "device-id" - dat, err := os.ReadFile(path.Join(dataDir, fname)) - if err != nil { - return "", fmt.Errorf("could not read file with device-id: %w", err) - } - id := strings.TrimRight(string(dat), "\n") - return id, nil -} diff --git a/internal/output/output.go b/internal/output/output.go index 2799e57..e3fd15f 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -41,11 +41,13 @@ var msgDeamonNotRunning = `Resh-daemon didn't respond - it's probably not runnin -> Try restarting this terminal window to bring resh-daemon back up -> If the problem persists you can check resh-daemon logs: ~/.local/share/resh/log.json (or ~/$XDG_DATA_HOME/resh/log.json) -> You can create an issue at: https://github.com/curusarn/resh/issues + ` var msgVersionMismatch = `This terminal session was started with different resh version than is installed now. It looks like you updated resh and didn't restart this terminal. -> Restart this terminal window to fix that + ` func (f *Output) ErrorDaemonNotRunning(err error) { @@ -67,7 +69,7 @@ func (f *Output) ErrorVersionMismatch(installedVer, terminalVer string) { } func (f *Output) FatalVersionMismatch(installedVer, terminalVer string) { - fmt.Fprintf(os.Stderr, "%s: %s\n\n(installed version: %s, this terminal version: %s)", + fmt.Fprintf(os.Stderr, "%s: %s\n(installed version: %s, this terminal version: %s)\n", f.ErrPrefix, msgVersionMismatch, installedVer, terminalVer) f.Logger.Fatal("Version mismatch", zap.String("installed", installedVer), diff --git a/internal/recconv/recconv.go b/internal/recconv/recconv.go index 36cf605..ec5c3da 100644 --- a/internal/recconv/recconv.go +++ b/internal/recconv/recconv.go @@ -1,6 +1,8 @@ package recconv import ( + "fmt" + "github.com/curusarn/resh/internal/record" ) @@ -10,24 +12,24 @@ func LegacyToV1(r *record.Legacy) *record.V1 { // Flags: 0, - DeviceID: r.MachineID, - SessionID: r.SessionID, - RecordID: r.RecordID, - CmdLine: r.CmdLine, ExitCode: r.ExitCode, + DeviceID: r.ReshUUID, + SessionID: r.SessionID, + RecordID: r.RecordID, + Home: r.Home, Pwd: r.Pwd, RealPwd: r.RealPwd, - Logname: r.Login, - Hostname: r.Host, + // Logname: r.Login, + Device: r.Host, GitOriginRemote: r.GitOriginRemote, - Time: r.RealtimeBefore, - Duration: r.RealtimeDuration, + Time: fmt.Sprintf("%.4f", r.RealtimeBefore), + Duration: fmt.Sprintf("%.4f", r.RealtimeDuration), PartOne: r.PartOne, PartsNotMerged: !r.PartsMerged, diff --git a/internal/recload/recload.go b/internal/recload/recload.go deleted file mode 100644 index 0db3e3c..0000000 --- a/internal/recload/recload.go +++ /dev/null @@ -1 +0,0 @@ -package recload diff --git a/internal/record/v1.go b/internal/record/v1.go index 1620ccd..6585051 100644 --- a/internal/record/v1.go +++ b/internal/record/v1.go @@ -6,24 +6,26 @@ type V1 struct { // FIXME: is this the best way? .. what about string, separate fields, or something similar Flags int `json:"flags"` + // cmdline, exitcode + CmdLine string `json:"cmdLine"` + ExitCode int `json:"exitCode"` + DeviceID string `json:"deviceID"` SessionID string `json:"sessionID"` // can we have a shorter uuid for record RecordID string `json:"recordID"` - // cmdline, exitcode - CmdLine string `json:"cmdLine"` - ExitCode int `json:"exitCode"` - // paths // TODO: Do we need both pwd and real pwd? Home string `json:"home"` Pwd string `json:"pwd"` RealPwd string `json:"realPwd"` - // hostname + lognem (not sure if we actually need logname) - Logname string `json:"logname"` - Hostname string `json:"hostname"` + // hostname + logname (not sure if we actually need logname) + // Logname string `json:"logname"` + // Device is usually hostname but not stricly hostname + // It can be configured in RESH configuration + Device string `json:"device"` // git info // origin is the most important @@ -45,8 +47,10 @@ type V1 struct { // Shlvl int `json:"shlvl"` // time (before), duration of command - Time float64 `json:"time"` - Duration float64 `json:"duration"` + // time and duration are strings because we don't want unnecessary precision when they get serialized into json + // we could implement custom (un)marshalling but I don't see downsides of directly representing the values as strings + Time string `json:"time"` + Duration string `json:"duration"` // these look like internal stuff diff --git a/internal/recordint/searchapp.go b/internal/recordint/searchapp.go index a80b018..1b5f26d 100644 --- a/internal/recordint/searchapp.go +++ b/internal/recordint/searchapp.go @@ -2,6 +2,7 @@ package recordint import ( "net/url" + "strconv" "strings" giturls "github.com/whilp/git-urls" @@ -37,17 +38,19 @@ func NewSearchAppFromCmdLine(cmdLine string) SearchApp { // NewCliRecord from EnrichedRecord func NewSearchApp(r *Indexed) SearchApp { // TODO: we used to validate records with recutil.Validate() + // TODO: handle this error + time, _ := strconv.ParseFloat(r.Rec.Time, 64) return SearchApp{ IsRaw: false, SessionID: r.Rec.SessionID, CmdLine: r.Rec.CmdLine, - Host: r.Rec.Hostname, + Host: r.Rec.Device, Pwd: r.Rec.Pwd, Home: r.Rec.Home, // TODO: is this the right place to normalize the git remote GitOriginRemote: normalizeGitRemote(r.Rec.GitOriginRemote), ExitCode: r.Rec.ExitCode, - Time: r.Rec.Time, + Time: time, Idx: r.Idx, } diff --git a/scripts/hooks.sh b/scripts/hooks.sh index 20d8c9e..b331ec0 100644 --- a/scripts/hooks.sh +++ b/scripts/hooks.sh @@ -19,10 +19,7 @@ __resh_collect() { # non-posix local __RESH_SHLVL="$SHLVL" local __RESH_GIT_REMOTE; __RESH_GIT_REMOTE="$(git remote get-url origin 2>/dev/null)" - local __RESH_GIT_REMOTE_EXIT_CODE=$? - local __RESH_PID="$$" # current pid - # time # __RESH_RT_BEFORE="$EPOCHREALTIME" __RESH_RT_BEFORE=$(__resh_get_epochrealtime) @@ -36,22 +33,23 @@ __resh_collect() { fi elif [ "$__RESH_REVISION" != "$(resh-collect -revision)" ]; then # shellcheck source=shellrc.sh - source ~/.resh/shellrc + source ~/.resh/shellrc if [ "$__RESH_REVISION" != "$(resh-collect -revision)" ]; then echo "RESH WARNING: You probably just updated RESH - PLEASE RESTART OR RELOAD THIS TERMINAL SESSION (resh revision: $(resh-collect -revision); resh revision of this terminal session: ${__RESH_REVISION})" fi fi + # TODO: change how resh-uuid is read if [ "$__RESH_VERSION" = "$(resh-collect -version)" ] && [ "$__RESH_REVISION" = "$(resh-collect -revision)" ]; then resh-collect -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ -shell "$__RESH_SHELL" \ + -device "$__RESH_HOST" \ + -deviceID "$(cat ~/.resh/resh-uuid 2>/dev/null)" \ -sessionID "$__RESH_SESSION_ID" \ -recordID "$__RESH_RECORD_ID" \ -home "$__RESH_HOME" \ - -logname "$__RESH_LOGIN" \ -pwd "$__RESH_PWD" \ -sessionPID "$__RESH_SESSION_PID" \ - -hostname "$__RESH_HOST" \ -shlvl "$__RESH_SHLVL" \ -gitRemote "$__RESH_GIT_REMOTE" \ -time "$__RESH_RT_BEFORE" \ diff --git a/scripts/install.sh b/scripts/install.sh index 336c4cf..d05a648 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -31,7 +31,7 @@ if [ "$bash_too_old" = true ]; then if [ "$login_shell" = bash ]; then echo " > Your bash version is old." echo " > Bash is also your login shell." - echo " > Updating to bash 4.3+ is strongly RECOMMENDED!" + echo " > Updating to bash 4.3+ is STRONGLY RECOMMENDED!" else echo " > Your bash version is old" echo " > Bash is not your login shell so it should not be an issue." @@ -52,7 +52,7 @@ else if [ "$login_shell" = zsh ]; then echo " > Your zsh version is old." echo " > Zsh is also your login shell." - echo " > Updating to Zsh 5.0+ is strongly RECOMMENDED!" + echo " > Updating to Zsh 5.0+ is STRONGLY RECOMMENDED!" else echo " > Your zsh version is old" echo " > Zsh is not your login shell so it should not be an issue." @@ -93,12 +93,12 @@ fi # # shellcheck disable=2034 # read -r x -echo echo "Backing up previous installation" -# TODO: ~/.resh -> XDG_DATA/resh/rollback/ -# TODO: ~/XDG_DATA/resh/history.reshjson -> XDG_DATA/resh/rollback/ +#./bin/resh-install-utils backup +# TODO: ~/.resh -> XDG_DATA_HOME/resh/rollback/ +# TODO: ~/XDG_DATA_HOME/resh/history.reshjson -> XDG_DATA/resh/rollback/ # TODO: what about legacy history locations -# TODO: ~/XDG_DATA/resh/log.json -> XDG_DATA/resh/rollback/ +# TODO: ~/XDG_DATA_HOME/resh/log.json -> XDG_DATA/resh/rollback/ echo "Cleaning up installation directory ..." rm ~/.resh/bin/* 2>/dev/null ||: @@ -107,6 +107,8 @@ rm ~/.resh/* 2>/dev/null 2>/dev/null ||: # backward compatibility: We have a new location for resh history file [ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json +#[ ! -f ~/.resh_history.json ] || mv ~/.resh_history.json $XDG .resh_history.json + echo "Creating directories ..." mkdir_if_not_exists() { @@ -117,8 +119,6 @@ mkdir_if_not_exists() { mkdir_if_not_exists ~/.resh mkdir_if_not_exists ~/.resh/bin -mkdir_if_not_exists ~/.resh/bash_completion.d -mkdir_if_not_exists ~/.resh/zsh_completion.d mkdir_if_not_exists ~/.config echo "Copying files ..." @@ -138,7 +138,7 @@ cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid cp -f bin/resh-{daemon,cli,control,collect,postcollect,session-init,config} ~/.resh/bin/ echo "Creating/updating config file ..." -./bin/resh-config-setup +./bin/resh-install-utils migrate-config echo "Finishing up ..." # Adding resh shellrc to .bashrc ... @@ -146,14 +146,14 @@ if [ ! -f ~/.bashrc ]; then touch ~/.bashrc fi grep -q '[[ -f ~/.resh/shellrc ]] && source ~/.resh/shellrc' ~/.bashrc ||\ - echo -e '\n[[ -f ~/.resh/shellrc ]] && source ~/.resh/shellrc # this line was added by RESH (Rich Enchanced Shell History)' >> ~/.bashrc + echo -e '\n[[ -f ~/.resh/shellrc ]] && source ~/.resh/shellrc # this line was added by RESH (Rich Enhanced Shell History)' >> ~/.bashrc # Adding bash-preexec to .bashrc ... grep -q '[[ -f ~/.bash-preexec.sh ]] && source ~/.bash-preexec.sh' ~/.bashrc ||\ - echo -e '\n[[ -f ~/.bash-preexec.sh ]] && source ~/.bash-preexec.sh # this line was added by RESH (Rich Enchanced Shell History)' >> ~/.bashrc + echo -e '\n[[ -f ~/.bash-preexec.sh ]] && source ~/.bash-preexec.sh # this line was added by RESH (Rich Enhanced Shell History)' >> ~/.bashrc # Adding resh shellrc to .zshrc ... if [ -f ~/.zshrc ]; then grep -q '[ -f ~/.resh/shellrc ] && source ~/.resh/shellrc' ~/.zshrc ||\ - echo -e '\n[ -f ~/.resh/shellrc ] && source ~/.resh/shellrc # this line was added by RESH (Rich Enchanced Shell History)' >> ~/.zshrc + echo -e '\n[ -f ~/.resh/shellrc ] && source ~/.resh/shellrc # this line was added by RESH (Rich Enhanced Shell History)' >> ~/.zshrc fi # Deleting zsh completion cache - for future use @@ -178,9 +178,6 @@ if [ -f ~/.resh/resh.pid ]; then else pkill -SIGTERM "resh-daemon" || true fi -# daemon uses xdg path variables -# FIXME: this does not exist anymore -#__resh_set_xdg_home_paths __resh_run_daemon @@ -195,6 +192,7 @@ info="---- Scroll down using arrow keys ---- ##################################### " +# FIMXE: update info - resh history path info="$info RESH SEARCH APPLICATION = Redesigned reverse search that actually works @@ -214,7 +212,7 @@ CHECK FOR UPDATES HISTORY Your resh history will be recorded to '~/.resh_history.json' Look at it using e.g. following command (you might need to install jq) - $ tail -f ~/.resh_history.json | jq + $ cat ~/.resh_history.json | sed 's/^v[^{]*{/{/' | jq . ISSUES & FEEDBACK Please report issues to: https://github.com/curusarn/resh/issues diff --git a/scripts/shellrc.sh b/scripts/shellrc.sh index 9092ff6..cf56daf 100644 --- a/scripts/shellrc.sh +++ b/scripts/shellrc.sh @@ -12,67 +12,45 @@ PATH=$PATH:~/.resh/bin # shellcheck source=reshctl.sh . ~/.resh/reshctl.sh -__RESH_MACOS=0 -__RESH_LINUX=0 -__RESH_UNAME=$(uname) - -if [ "$__RESH_UNAME" = "Darwin" ]; then - __RESH_MACOS=1 -elif [ "$__RESH_UNAME" = "Linux" ]; then - __RESH_LINUX=1 -else - echo "resh PANIC unrecognized OS" -fi - if [ -n "${ZSH_VERSION-}" ]; then # shellcheck disable=SC1009 __RESH_SHELL="zsh" __RESH_HOST="$HOST" - __RESH_HOSTTYPE="$CPUTYPE" - __resh_zsh_completion_init elif [ -n "${BASH_VERSION-}" ]; then __RESH_SHELL="bash" __RESH_HOST="$HOSTNAME" - __RESH_HOSTTYPE="$HOSTTYPE" - __resh_bash_completion_init else - echo "resh PANIC unrecognized shell" + echo "RESH PANIC: unrecognized shell - please report this to https://github.com/curusarn/resh/issues" fi -# posix +# TODO: read this from resh-specific file +# create that file during install +__RESH_DEVICE="$__RESH_HOST" __RESH_HOME="$HOME" -__RESH_LOGIN="$LOGNAME" -__RESH_SHELL_ENV="$SHELL" -__RESH_TERM="$TERM" - -# non-posix -__RESH_RT_SESSION=$(__resh_get_epochrealtime) -__RESH_OSTYPE="$OSTYPE" -__RESH_MACHTYPE="$MACHTYPE" # shellcheck disable=2155 export __RESH_VERSION=$(resh-collect -version) # shellcheck disable=2155 export __RESH_REVISION=$(resh-collect -revision) -# FIXME: this does not exist anymore -# __resh_set_xdg_home_paths - __resh_run_daemon [ "$(resh-config --key BindControlR)" = true ] && __resh_bind_control_R # block for anything we only want to do once per session # NOTE: nested shells are still the same session +# i.e. $__RESH_SESSION_ID will be set in nested shells if [ -z "${__RESH_SESSION_ID+x}" ]; then export __RESH_SESSION_ID; __RESH_SESSION_ID=$(__resh_get_uuid) export __RESH_SESSION_PID="$$" - # TODO add sesson time + __resh_reset_variables __resh_session_init fi # block for anything we only want to do once per shell +# NOTE: nested shells are new shells +# i.e. $__RESH_INIT_DONE will NOT be set in nested shells if [ -z "${__RESH_INIT_DONE+x}" ]; then preexec_functions+=(__resh_preexec) precmd_functions+=(__resh_precmd) diff --git a/scripts/util.sh b/scripts/util.sh index 8d616a5..37d7b1f 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -54,67 +54,16 @@ __resh_run_daemon() { fi if [ "$(uname)" = Darwin ]; then # hotfix - # gnohup resh-daemon 2>&1 & disown - gnohup resh-daemon & disown + gnohup resh-daemon >/dev/null 2>/dev/null & disown else # TODO: switch to nohup for consistency once you confirm that daemon is # not getting killed anymore on macOS - nohup resh-daemon & disown - #nohup resh-daemon 2>&1 & disown + nohup resh-daemon >/dev/null 2>/dev/null & disown #setsid resh-daemon 2>&1 & disown fi } -__resh_bash_completion_init() { - # primitive check to find out if bash_completions are installed - # skip completion init if they are not - _get_comp_words_by_ref >/dev/null 2>/dev/null - [[ $? == 127 ]] && return - . ~/.resh/bash_completion.d/_reshctl -} - -# TODO: redo this -__resh_zsh_completion_init() { - # NOTE: this is hacky - each completion needs to be added individually - # TODO: fix later - # fpath=(~/.resh/zsh_completion.d $fpath) - # we should be using fpath but that doesn't work well with oh-my-zsh - # so we are just adding it manually - # shellcheck disable=1090 - if typeset -f compdef >/dev/null 2>&1; then - source ~/.resh/zsh_completion.d/_reshctl && compdef _reshctl reshctl - else - # fallback I guess - fpath=(~/.resh/zsh_completion.d $fpath) - __RESH_zsh_no_compdef=1 - fi - - # TODO: test and use this - # NOTE: this is not how globbing works - # for f in ~/.resh/zsh_completion.d/_*; do - # source ~/.resh/zsh_completion.d/_$f && compdef _$f $f - # done -} - __resh_session_init() { - # posix - local __RESH_COLS="$COLUMNS" - local __RESH_LANG="$LANG" - local __RESH_LC_ALL="$LC_ALL" - # other LC ? - local __RESH_LINES="$LINES" - local __RESH_PWD="$PWD" - - # non-posix - local __RESH_SHLVL="$SHLVL" - - # pid - local __RESH_PID; __RESH_PID=$(__resh_get_pid) - - # time - local __RESH_TZ_BEFORE; __RESH_TZ_BEFORE=$(date +%z) - local __RESH_RT_BEFORE; __RESH_RT_BEFORE=$(__resh_get_epochrealtime) - if [ "$__RESH_VERSION" != "$(resh-session-init -version)" ]; then # shellcheck source=shellrc.sh source ~/.resh/shellrc @@ -136,5 +85,4 @@ __resh_session_init() { -sessionId "$__RESH_SESSION_ID" \ -sessionPid "$__RESH_SESSION_PID" fi - } From 9a632d4192f30645172440da4e3f9a4e01d83bcd Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 26 Oct 2022 01:28:29 +0200 Subject: [PATCH 23/35] install changes --- scripts/install.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index d05a648..81a1328 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -102,7 +102,7 @@ echo "Backing up previous installation" echo "Cleaning up installation directory ..." rm ~/.resh/bin/* 2>/dev/null ||: -rm ~/.resh/* 2>/dev/null 2>/dev/null ||: +rm ~/.resh/* 2>/dev/null ||: # TODO: put this behind version condition # backward compatibility: We have a new location for resh history file [ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json @@ -129,10 +129,6 @@ cp -f scripts/shellrc.sh ~/.resh/shellrc cp -f scripts/reshctl.sh scripts/widgets.sh scripts/hooks.sh scripts/util.sh ~/.resh/ cp -f scripts/rawinstall.sh ~/.resh/ -echo "Generating completions ..." -bin/resh-control completion bash > ~/.resh/bash_completion.d/_reshctl -bin/resh-control completion zsh > ~/.resh/zsh_completion.d/_reshctl - echo "Copying more files ..." cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid cp -f bin/resh-{daemon,cli,control,collect,postcollect,session-init,config} ~/.resh/bin/ @@ -160,6 +156,7 @@ fi # [ ! -e ~/.zcompdump ] || rm ~/.zcompdump # Final touch +# TODO: change touch ~/.resh_history.json # Generating resh-uuid ... @@ -203,16 +200,16 @@ RESH SEARCH APPLICATION = Redesigned reverse search that actually works Host, directories, git remote, and exit status is used to display relevant results first. At first, the search application will use the standard shell history without context. - All history recorded from now on will have context which will by the RESH SEARCH app. + All history recorded from now on will have context which will be used by the RESH SEARCH app. CHECK FOR UPDATES To check for (and install) updates use reshctl command: $ reshctl update HISTORY - Your resh history will be recorded to '~/.resh_history.json' + Your resh history will be recorded to '${XDG_DATA_HOME-~/.local/share}/resh/history/.reshjson' Look at it using e.g. following command (you might need to install jq) - $ cat ~/.resh_history.json | sed 's/^v[^{]*{/{/' | jq . + $ cat ${XDG_DATA_HOME-~/.local/share}/resh/history/.reshjson | sed 's/^v[^{]*{/{/' | jq . ISSUES & FEEDBACK Please report issues to: https://github.com/curusarn/resh/issues @@ -237,7 +234,7 @@ echo "All done!" echo "Thank you for using RESH" echo "Issues go here: https://github.com/curusarn/resh/issues" echo "Ctrl+R launches the RESH SEARCH app" -# echo "Do not forget to restart your terminal" + if [ -z "${__RESH_VERSION:-}" ]; then echo " ############################################################## # # From 3cefa0fdad75a5e1cacf113321318fa47b6d0ad2 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 26 Oct 2022 01:50:59 +0200 Subject: [PATCH 24/35] change config, remove data --- conf/config.toml | 6 +- data/sanitizer/copyright_information.md | 7 - data/sanitizer/whitelist.txt | 1195 ----------------------- 3 files changed, 1 insertion(+), 1207 deletions(-) delete mode 100644 data/sanitizer/copyright_information.md delete mode 100644 data/sanitizer/whitelist.txt diff --git a/conf/config.toml b/conf/config.toml index 52adf66..85109aa 100644 --- a/conf/config.toml +++ b/conf/config.toml @@ -1,5 +1 @@ -port = 2627 -sesswatchPeriodSeconds = 120 -sesshistInitHistorySize = 1000 -bindControlR = true -logVerbosity = info +configVersion = "v1" diff --git a/data/sanitizer/copyright_information.md b/data/sanitizer/copyright_information.md deleted file mode 100644 index abdbf33..0000000 --- a/data/sanitizer/copyright_information.md +++ /dev/null @@ -1,7 +0,0 @@ -# copyright information - -Whitelist contains content from variety of sources. - -Part of the whitelist (`./whitelist.txt`) is made of copyrighted content from [FileInfo.com](https://fileinfo.com/filetypes/common). - -This content was used with permission from FileInfo.com. diff --git a/data/sanitizer/whitelist.txt b/data/sanitizer/whitelist.txt deleted file mode 100644 index 180e9c3..0000000 --- a/data/sanitizer/whitelist.txt +++ /dev/null @@ -1,1195 +0,0 @@ - -! -- -. -.. -: -[ -[[ -]] -{ -} -3dm -3ds -3g2 -3gp -7z -accdb -add -addgnupghome -addgroup -addpart -addr2line -add-shell -adduser -agetty -ai -aif -alias -alternatives -apk -app -applydeltarpm -applygnupgdefaults -apt -apt-cache -apt-cdrom -apt-config -apt-get -apt-key -apt-mark -ar -arch -arpd -arping -as -asf -asm -asp -aspx -au -autoload -avi -awk -b -b2sum -badblocks -bak -base32 -base64 -basename -basenc -bash -bashbug -bashbug-64 -bat -bg -bin -bind -bindkey -bisect -blend -blkdeactivate -blkdiscard -blkid -blkzone -blockdev -bmp -boot -bootctl -br -branch -break -bridge -brotli -build-locale-archive -builtin -bunzip2 -busctl -bye -bz2 -bzcat -bzcmp -bzdiff -bzegrep -bzexe -bzfgrep -bzgrep -bzip2 -bzip2recover -bzless -bzmore -c -cab -cal -ca-legacy -caller -capsh -captoinfo -case -cat -catchsegv -cbr -cc -cd -cer -certutil -cfdisk -cfg -c++filt -cfm -cgi -chacl -chage -chardetect -chattr -chcon -chcpu -chdir -checkout -chfn -chgpasswd -chgrp -chkconfig -chmem -chmod -choom -chown -chpasswd -chroot -chrt -chsh -cksum -class -clear -clear_console -clock -clockdiff -clone -cmp -cmsutil -co -code -col -colcrt -colrm -column -com -combinedeltarpm -comm -command -commit -compadd -comparguments -compcall -compctl -compdescribe -compfiles -compgen -compgroups -complete -compopt -compquote -compset -comptags -comptry -compvalues -conf -continue -convert -coproc -coredumpctl -cp -cpgr -cpio -cpl -cpp -cppw -cracklib-check -cracklib-format -cracklib-packer -cracklib-unpacker -crdownload -create-cracklib-dict -crlutil -crx -cs -csplit -csr -css -csv -ctrlaltdel -ctstat -cue -cur -curl -cut -cvtsudoers -cz -dash -dat -date -db -db_archive -db_checkpoint -db_deadlock -db_dump -db_dump185 -dbf -db_hotbackup -db_load -db_log_verify -db_printlog -db_recover -db_replicate -db_stat -db_tuner -db_upgrade -dbus-binding-tool -dbus-broker -dbus-broker-launch -dbus-cleanup-sockets -dbus-daemon -dbus-monitor -dbus-run-session -dbus-send -dbus-test-tool -dbus-update-activation-environment -dbus-uuidgen -db_verify -dcr -dd -dds -de -deb -debconf -debconf-apt-progress -debconf-communicate -debconf-copydb -debconf-escape -debconf-set-selections -debconf-show -deb-systemd-helper -deb-systemd-invoke -debugfs -debuginfo-install -declare -delgroup -delpart -deluser -dem -depmod -deskthemepack -desktop -dev -devlink -df -dgawk -diff -diff3 -dir -dircolors -dirmngr -dirmngr-client -dirname -dirs -disable -disown -dll -dmesg -dmfilemapd -dmg -dmp -dmsetup -dmstats -dnf -dnf-3 -dnsdomainname -do -doc -docker -Dockerfile -docx -domainname -done -dpkg -dpkg-deb -dpkg-divert -dpkg-maintscript-helper -dpkg-preconfigure -dpkg-query -dpkg-reconfigure -dpkg-split -dpkg-statoverride -dpkg-trigger -dracut -drv -dtd -du -dumpe2fs -dwg -dwp -dxf -e2freefrag -e2fsck -e2image -e2label -e2mmpstatus -e2undo -e4crypt -e4defrag -easy_install-3.7 -echo -echotc -echoti -egrep -eject -elfedit -elif -else -emacs -emulate -enable -end -env -eps -esac -etc -eval -evmctl -ex -exe -exec -exit -expand -expiry -export -expr -factor -faillock -faillog -fallocate -false -fc -fdformat -fdisk -fetch -ffmpeg -fg -fgrep -fi -filefrag -fincore -find -findfs -findmnt -find-repos-of-install -fips-finish-install -fips-mode-setup -fish -fla -float -flock -flv -fmt -fnt -fold -fon -for -foreach -free -fsck -fsck.cramfs -fsck.ext2 -fsck.ext3 -fsck.ext4 -fsck.minix -fsfreeze -fstab-decode -fstrim -function -functions -g13 -g13-syshelp -gadget -gam -gapplication -gawk -gdbus -ged -gencat -genl -getcap -getconf -getent -getfacl -getln -getopt -getopts -getpcaps -getty -gif -gio -gio-launch-desktop -gio-querymodules-64 -git -github.com -glib-compile-schemas -glibc_post_upgrade.x86_64 -go -gpasswd -gpg -gpg2 -gpg-agent -gpgconf -gpg-connect-agent -gpg-error -gpgme-json -gpgparsemail -gpgsplit -gpgv -gpgv2 -gpg-wks-server -gpg-zip -gprof -gpx -grep -groupadd -groupdel -groupmems -groupmod -groups -grpck -grpconv -grpunconv -gsettings -gtar -gunzip -gz -gzexe -gzip -h -halt -hardlink -hash -head -heic -help -hexdump -history -home -hostid -hostname -hostnamectl -hqx -htm -html -http -https -hwclock -i386 -icns -ico -iconv -iconvconfig -iconvconfig.x86_64 -ics -id -idn -if -ifenslave -iff -igawk -in -indd -info -infocmp -infokey -infotocap -ini -init -initctl -insmod -install -install-info -installkernel -integer -invoke-rc.d -ionice -ip -ipcmk -ipcrm -ipcs -ir -ischroot -iso -isosize -it -jar -java -jobs -join -journalctl -jpg -jq -js -json -jsp -kernel-install -key -keychain -kill -killall5 -kml -kmod -kmz -kpartx -ksp -kss -kwd -last -lastb -lastlog -lchage -lchfn -lchsh -ld -ldattach -ld.bfd -ldconfig -ldconfig.real -ldd -ld.gold -let -lgroupadd -lgroupdel -lgroupmod -lib -lib64 -lid -limit -link -linux32 -linux64 -ln -lnewusers -lnk -lnstat -local -locale -locale-check -localectl -localedef -localhost -log -logger -login -loginctl -logname -logout -logsave -look -losetup -lost+found -lpasswd -ls -lsattr -lsblk -lscpu -lsinitrd -lsipc -lslocks -lslogins -lsmem -lsmod -lsns -lua -luac -luseradd -luserdel -lusermod -lz4 -lz4c -lz4cat -m -m3u -m4a -m4p -m4v -machinectl -make -makedb -makedeltarpm -make-dummy-cert -Makefile -man -mapfile -master -mawk -max -mcookie -md5 -md5sum -md5sums -md5sum.textutils -mdb -mdf -media -merge -mesg -mid -mim -mkdict -mkdir -mke2fs -mkfifo -mkfs -mkfs.bfs -mkfs.cramfs -mkfs.ext2 -mkfs.ext3 -mkfs.ext4 -mkfs.minix -mkhomedir_helper -mkinitrd -mklost+found -mknod -mkpasswd -mkswap -mktemp -mnt -mo -modinfo -modprobe -modulemd-validator-v1 -modutil -more -mount -mountpoint -mov -mp3 -mp4 -mpa -mpg -msg -msi -mv -namei -nawk -needs-restarting -nes -net -networkctl -newgidmap -newgrp -newuidmap -newusers -nice -nisdomainname -nl -nm -no -nocorrect -noglob -nohup -nologin -nproc -nsenter -nstat -numfmt -o -obj -objcopy -objdump -od -odt -ogg -oldfind -openssl -opt -org -origin -otf -p11-kit -package-cleanup -packer -pager -pages -pam-auth-update -pam_console_apply -pam_extrausers_chkpwd -pam_extrausers_update -pam_getenv -pam_tally -pam_tally2 -pam_timestamp_check -part -partx -passwd -paste -patch -pathchk -pct -pdb -pdf -perl -perl5.26.1 -perl5.28.1 -pgawk -pgrep -php -phps -phtml -pidof -pinentry -pinentry-curses -ping -ping4 -ping6 -pinky -pip-3 -pip3 -pip-3.7 -pip3.7 -pivot_root -pk12util -pkg -pkg-config -pkill -pkl -pl -pldd -pls -plugin -pmap -png -policy-rc.d -popd -portablectl -pov -poweroff -pps -ppt -pptx -pr -prf -print -printenv -printf -private -prlimit -proc -properties -ps -psd -pspimage -ptx -pull -push -pushd -pushln -pwck -pwconv -pwd -pwdx -pwhistory_helper -pwmake -pwscore -pwunconv -py -pyc -pydoc -pydoc3 -pydoc3.7 -pyo -python -python2 -python2.7 -python3 -python3.7 -python3.7m -pyvenv -pyvenv-3.7 -r -ranlib -rar -raw -rbash -rc -rdf -rdisc -rdma -read -readarray -readelf -readlink -readonly -readprofile -realpath -rebase -reboot -rehash -remove-shell -rename -rename.ul -renew-dummy-cert -renice -repeat -repoclosure -repodiff -repo-graph -repomanage -repoquery -repo-rss -reposync -repotrack -reset -resh -resize2fs -resizepart -resolvconf -resolvectl -return -rev -rfkill -rgrep -rm -rmdir -rmmod -rmt -rmt-tar -rom -root -routef -routel -rpcgen -rpm -rpm2archive -rpm2cpio -rpmdb -rpmdumpheader -rpmkeys -rpmquery -rpmverify -rss -rtacct -rtcwake -rtf -rtmon -rtstat -ru -run -runcon -run-help -runlevel -run-parts -runuser -rvi -rview -s -sasldblistusers2 -saslpasswd2 -sav -savelog -sbin -sched -script -scriptreplay -sdf -sdiff -sed -sefcontext_compile -select -select-editor -sensible-browser -sensible-editor -sensible-pager -seq -service -set -setarch -setcap -setfacl -setopt -setpriv -setsid -setterm -setup-nsssysinit -setup-nsssysinit.sh -sfdisk -sg -sh -sha1sum -sha224sum -sha256sum -sha384sum -sha512sum -shadowconfig -share -sh.distrib -shift -shopt -show -show-changed-rco -show-installed -shred -shuf -shutdown -signtool -signver -sitx -size -skill -slabtop -sleep -sln -snice -so -sort -sotruss -source -split -sprof -sql -sqlite3 -srt -srv -ss -ssh -ssltap -start-stop-daemon -stat -status -stdbuf -strings -strip -stty -su -sudo -sudoedit -sudoreplay -sulogin -sum -suspend -svg -swaplabel -swapoff -swapon -swf -swift -switch_root -sync -sys -sysctl -systemctl -systemd-analyze -systemd-ask-password -systemd-cat -systemd-cgls -systemd-cgtop -systemd-coredumpctl -systemd-delta -systemd-detect-virt -systemd-escape -systemd-firstboot -systemd-hwdb -systemd-id128 -systemd-inhibit -systemd-loginctl -systemd-machine-id-setup -systemd-mount -systemd-notify -systemd-nspawn -systemd-path -systemd-resolve -systemd-run -systemd-socket-activate -systemd-stdio-bridge -systemd-sysusers -systemd-tmpfiles -systemd-tty-ask-password-agent -systemd-umount -tabs -tac -tag -tail -tailf -tar -tarcat -taskset -tax2016 -tax2018 -tc -tee -telinit -tempfile -test -testgdbm -tex -tga -tgz -then -thm -tic -tif -tiff -tig -time -timedatectl -timeout -times -tipc -tload -tmp -toast -toe -top -torrent -touch -tput -tr -tracepath -tracepath6 -trap -true -truncate -trust -tset -tsort -ttf -tty -ttyctl -tune2fs -txt -type -typeset -tzconfig -tzselect -udevadm -uk -ul -ulimit -umask -umount -unalias -uname -uname26 -unbound-anchor -uncompress -unexpand -unfunction -unhash -uniq -unix_chkpwd -unix_update -unlimit -unlink -unlz4 -unminimize -unset -unsetopt -unshare -until -unxz -update-alternatives -update-ca-trust -update-crypto-policies -update-mime-database -update-passwd -update-rc.d -uptime -urlgrabber -useradd -userdel -usermod -users -usr -utmpdump -uue -uuidgen -uuidparse -Vagrantfile -var -vared -vb -vcd -vcf -vcxproj -vdir -verifytree -vi -view -vigr -vim -vipw -visudo -vlc -vmstat -vob -w -wait -wall -watch -watchgnupg -wav -wc -wdctl -weak-modules -whence -where -whereis -which -which-command -while -who -whoami -wipefs -wma -wmv -wpd -w.procps -wps -write -wsf -x86_64 -xargs -xbel -xcodeproj -xhtml -xlr -xls -xlsx -xml -xmlcatalog -xmllint -xmlwf -xpm -xsd -xsl -xz -xzcat -xzcmp -xzdec -xzdiff -xzegrep -xzfgrep -xzgrep -xzless -xzmore -yaourt -yes -ypdomainname -yum -yum-builddep -yum-complete-transaction -yum-config-manager -yumdb -yum-debug-dump -yum-debug-restore -yumdownloader -yum-groups-manager -yuv -Z -zcat -zcmp -zcompile -zdiff -zdump -zegrep -zfgrep -zforce -zformat -zgrep -zic -zip -zipx -zle -zless -zmodload -zmore -znew -zparseopts -zramctl -zregexparse -zsh -zstyle From e359d53885d49de8511c71be3392addb68f5c256 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 1 Nov 2022 17:11:29 +0100 Subject: [PATCH 25/35] Make record package public --- {internal/record => record}/legacy.go | 0 {internal/record => record}/record.go | 0 {internal/record => record}/v1.go | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {internal/record => record}/legacy.go (100%) rename {internal/record => record}/record.go (100%) rename {internal/record => record}/v1.go (100%) diff --git a/internal/record/legacy.go b/record/legacy.go similarity index 100% rename from internal/record/legacy.go rename to record/legacy.go diff --git a/internal/record/record.go b/record/record.go similarity index 100% rename from internal/record/record.go rename to record/record.go diff --git a/internal/record/v1.go b/record/v1.go similarity index 100% rename from internal/record/v1.go rename to record/v1.go From 6ec71ac8bc8d30d94b36bf627ee2135634638a9f Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 1 Nov 2022 17:16:41 +0100 Subject: [PATCH 26/35] update imports --- cmd/collect/main.go | 2 +- cmd/postcollect/main.go | 2 +- internal/histfile/histfile.go | 2 +- internal/histio/histio.go | 2 +- internal/recconv/recconv.go | 2 +- internal/recio/read.go | 2 +- internal/recio/write.go | 2 +- internal/recordint/collect.go | 2 +- internal/recordint/indexed.go | 2 +- internal/recutil/recutil.go | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/collect/main.go b/cmd/collect/main.go index 1974870..230fda8 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -9,8 +9,8 @@ import ( "github.com/curusarn/resh/internal/collect" "github.com/curusarn/resh/internal/logger" "github.com/curusarn/resh/internal/output" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" "go.uber.org/zap" // "os/exec" diff --git a/cmd/postcollect/main.go b/cmd/postcollect/main.go index 2023a6a..a187808 100644 --- a/cmd/postcollect/main.go +++ b/cmd/postcollect/main.go @@ -9,8 +9,8 @@ import ( "github.com/curusarn/resh/internal/collect" "github.com/curusarn/resh/internal/logger" "github.com/curusarn/resh/internal/output" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" "go.uber.org/zap" // "os/exec" diff --git a/internal/histfile/histfile.go b/internal/histfile/histfile.go index 9eea14d..33a2279 100644 --- a/internal/histfile/histfile.go +++ b/internal/histfile/histfile.go @@ -9,10 +9,10 @@ import ( "github.com/curusarn/resh/internal/histcli" "github.com/curusarn/resh/internal/histlist" "github.com/curusarn/resh/internal/recio" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" "github.com/curusarn/resh/internal/records" "github.com/curusarn/resh/internal/recutil" + "github.com/curusarn/resh/record" "go.uber.org/zap" ) diff --git a/internal/histio/histio.go b/internal/histio/histio.go index 3bc7c68..d541f9d 100644 --- a/internal/histio/histio.go +++ b/internal/histio/histio.go @@ -3,8 +3,8 @@ package histio import ( "path" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" "go.uber.org/zap" ) diff --git a/internal/recconv/recconv.go b/internal/recconv/recconv.go index ec5c3da..7690ec4 100644 --- a/internal/recconv/recconv.go +++ b/internal/recconv/recconv.go @@ -3,7 +3,7 @@ package recconv import ( "fmt" - "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/record" ) func LegacyToV1(r *record.Legacy) *record.V1 { diff --git a/internal/recio/read.go b/internal/recio/read.go index 304a4bd..b71cbb6 100644 --- a/internal/recio/read.go +++ b/internal/recio/read.go @@ -9,8 +9,8 @@ import ( "strings" "github.com/curusarn/resh/internal/recconv" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" "go.uber.org/zap" ) diff --git a/internal/recio/write.go b/internal/recio/write.go index 831a146..1ff4506 100644 --- a/internal/recio/write.go +++ b/internal/recio/write.go @@ -5,8 +5,8 @@ import ( "fmt" "os" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" ) // TODO: better errors diff --git a/internal/recordint/collect.go b/internal/recordint/collect.go index 645efe4..1a7d6a7 100644 --- a/internal/recordint/collect.go +++ b/internal/recordint/collect.go @@ -1,6 +1,6 @@ package recordint -import "github.com/curusarn/resh/internal/record" +import "github.com/curusarn/resh/record" type Collect struct { // record merging diff --git a/internal/recordint/indexed.go b/internal/recordint/indexed.go index 6d21870..950cadf 100644 --- a/internal/recordint/indexed.go +++ b/internal/recordint/indexed.go @@ -1,6 +1,6 @@ package recordint -import "github.com/curusarn/resh/internal/record" +import "github.com/curusarn/resh/record" // Indexed record allows us to find records in history file in order to edit them type Indexed struct { diff --git a/internal/recutil/recutil.go b/internal/recutil/recutil.go index e2b3c85..53b1cff 100644 --- a/internal/recutil/recutil.go +++ b/internal/recutil/recutil.go @@ -3,8 +3,8 @@ package recutil import ( "errors" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" ) // TODO: reintroduce validation From e8072597c87e3b05df05321b8814871df0344da2 Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Sun, 30 Oct 2022 19:26:47 +0100 Subject: [PATCH 27/35] config support for sync connector --- cmd/config/main.go | 11 +---------- internal/cfg/cfg.go | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cmd/config/main.go b/cmd/config/main.go index eaa68b8..a97536b 100644 --- a/cmd/config/main.go +++ b/cmd/config/main.go @@ -35,7 +35,7 @@ func main() { *configKey = strings.ToLower(*configKey) switch *configKey { case "bindcontrolr": - printBoolNormalized(config.BindControlR) + fmt.Println(config.BindControlR) case "port": fmt.Println(config.Port) case "sesswatchperiodseconds": @@ -47,12 +47,3 @@ func main() { os.Exit(1) } } - -// this might be unnecessary but I'm too lazy to look it up -func printBoolNormalized(x bool) { - if x { - fmt.Println("true") - } else { - fmt.Println("false") - } -} diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 76bd73b..01ea46e 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -29,6 +29,8 @@ type configFile struct { // deprecated in v1 BindArrowKeysBash *bool BindArrowKeysZsh *bool + + SyncConnectorAddress *string } // Config returned by this package to be used in the rest of the project @@ -47,13 +49,22 @@ type Config struct { // SessionWatchPeriodSeconds is how often should daemon check if terminal // sessions are still alive // There is not much need to adjust the value both memory overhead of watched sessions - // and the CPU overhead of chacking them are relatively low + // and the CPU overhead of checking them are relatively low SessionWatchPeriodSeconds uint // ReshHistoryMinSize is how large resh history needs to be for // daemon to ignore standard shell history files - // Ignoring standard shell history gives us more consistent experience + // Ignoring standard shell history gives us more consistent experience, // but you can increase this to something large to see standard shell history in RESH search ReshHistoryMinSize int + + // SyncConnectorAddress used by the daemon to connect to the Sync Connector + // examples: + // - localhost:1234 + // - http://localhost:1234 + // - 192.168.1.1:1324 + // - https://domain.tld + // - https://domain.tld/resh + SyncConnectorAddress *string } // defaults for config @@ -166,6 +177,8 @@ func processAndFillDefaults(configF *configFile) (Config, error) { config.BindControlR = *configF.BindControlR } + config.SyncConnectorAddress = configF.SyncConnectorAddress + return config, err } From ad4784de7d66bfe992231c73c1af26141f40e3c5 Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Mon, 31 Oct 2022 18:47:40 +0100 Subject: [PATCH 28/35] initial configurable Sync Connector stub, shared history object --- cmd/daemon/dump.go | 12 ++---- cmd/daemon/main.go | 2 +- cmd/daemon/run-server.go | 24 +++++++++-- internal/cfg/cfg.go | 20 ++++++++- internal/histcli/histcli.go | 23 ++++++++--- internal/histfile/histfile.go | 14 ++----- internal/histio/histio.go | 2 +- internal/histlist/histlist.go | 2 +- internal/searchapp/test.go | 2 +- internal/syncconnector/reader.go | 12 ++++++ internal/syncconnector/syncconnector.go | 54 +++++++++++++++++++++++++ internal/syncconnector/writer.go | 49 ++++++++++++++++++++++ 12 files changed, 185 insertions(+), 31 deletions(-) create mode 100644 internal/syncconnector/reader.go create mode 100644 internal/syncconnector/syncconnector.go create mode 100644 internal/syncconnector/writer.go diff --git a/cmd/daemon/dump.go b/cmd/daemon/dump.go index b3a154f..c488996 100644 --- a/cmd/daemon/dump.go +++ b/cmd/daemon/dump.go @@ -2,17 +2,17 @@ package main import ( "encoding/json" + "github.com/curusarn/resh/internal/histcli" "io/ioutil" "net/http" - "github.com/curusarn/resh/internal/histfile" "github.com/curusarn/resh/internal/msg" "go.uber.org/zap" ) type dumpHandler struct { - sugar *zap.SugaredLogger - histfileBox *histfile.Histfile + sugar *zap.SugaredLogger + history *histcli.Histcli } func (h *dumpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -35,12 +35,8 @@ func (h *dumpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } sugar.Debugw("Getting records to send ...") - fullRecords := h.histfileBox.DumpCliRecords() - if err != nil { - sugar.Errorw("Error when getting records", "error", err) - } - resp := msg.CliResponse{Records: fullRecords.List} + resp := msg.CliResponse{Records: h.history.Dump()} jsn, err = json.Marshal(&resp) if err != nil { sugar.Errorw("Error when marshaling", "error", err) diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index 6a6feff..fec397f 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -126,7 +126,7 @@ func (d *daemon) killDaemon(pidfile string) error { "PIDFile", pidfile, "error", err) } - d.sugar.Infow("Succesfully read PID file", "contents", string(dat)) + d.sugar.Infow("Successfully read PID file", "contents", string(dat)) pid, err := strconv.Atoi(strings.TrimSuffix(string(dat), "\n")) if err != nil { return fmt.Errorf("could not parse PID file contents: %w", err) diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index afe77a1..fcf1828 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -1,6 +1,8 @@ package main import ( + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/syncconnector" "net/http" "os" "strconv" @@ -33,6 +35,8 @@ func (s *Server) Run() { shutdown := make(chan string) + history := histcli.New() + // histfile histfileRecords := make(chan recordint.Collect) recordSubscribers = append(recordSubscribers, histfileRecords) @@ -42,13 +46,14 @@ func (s *Server) Run() { signalSubscribers = append(signalSubscribers, histfileSignals) maxHistSize := 10000 // lines minHistSizeKB := 2000 // roughly lines - histfileBox := histfile.New(s.sugar, histfileRecords, histfileSessionsToDrop, + histfile.New(s.sugar, histfileRecords, histfileSessionsToDrop, s.reshHistoryPath, s.bashHistoryPath, s.zshHistoryPath, maxHistSize, minHistSizeKB, - histfileSignals, shutdown) + histfileSignals, shutdown, history) // sesswatch sesswatchRecords := make(chan recordint.Collect) + // TODO: add sync connector subscriber recordSubscribers = append(recordSubscribers, sesswatchRecords) sesswatchSessionsToWatch := make(chan recordint.SessionInit) sessionInitSubscribers = append(sessionInitSubscribers, sesswatchSessionsToWatch) @@ -65,7 +70,7 @@ func (s *Server) Run() { mux.Handle("/status", &statusHandler{sugar: s.sugar}) mux.Handle("/record", &recordHandler{sugar: s.sugar, subscribers: recordSubscribers}) mux.Handle("/session_init", &sessionInitHandler{sugar: s.sugar, subscribers: sessionInitSubscribers}) - mux.Handle("/dump", &dumpHandler{sugar: s.sugar, histfileBox: histfileBox}) + mux.Handle("/dump", &dumpHandler{sugar: s.sugar, history: history}) server := &http.Server{ Addr: "localhost:" + strconv.Itoa(s.config.Port), @@ -77,6 +82,19 @@ func (s *Server) Run() { } go server.ListenAndServe() + s.sugar.Infow("", "sync_addr", s.config.SyncConnectorAddress) + if s.config.SyncConnectorAddress != nil { + sc, err := syncconnector.New(s.sugar, *s.config.SyncConnectorAddress, s.config.SyncConnectorAuthToken, s.config.SyncConnectorPullPeriodSeconds, history) + if err != nil { + s.sugar.Errorw("Sync Connector init failed", "error", err) + } else { + s.sugar.Infow("Initialized Sync Connector", "Sync Connector", sc) + } + // TODO: load sync connector data + // TODO: load sync connector data + // TODO: send connector data periodically (record by record / or batch) + } + // signalhandler - takes over the main goroutine so when signal handler exists the whole program exits signalhandler.Run(s.sugar, signalSubscribers, shutdown, server) } diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 01ea46e..e9a47cb 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -30,7 +30,9 @@ type configFile struct { BindArrowKeysBash *bool BindArrowKeysZsh *bool - SyncConnectorAddress *string + SyncConnectorAddress *string + SyncConnectorAuthToken *string + SyncConnectorPullPeriodSeconds *int } // Config returned by this package to be used in the rest of the project @@ -65,6 +67,12 @@ type Config struct { // - https://domain.tld // - https://domain.tld/resh SyncConnectorAddress *string + + // SyncConnectorAuthToken used by the daemon to authenticate with the Sync Connector + SyncConnectorAuthToken string + + // SyncConnectorPullPeriodSeconds how often should Resh daemon download history from Sync Connector + SyncConnectorPullPeriodSeconds int } // defaults for config @@ -76,6 +84,8 @@ var defaults = Config{ Debug: false, SessionWatchPeriodSeconds: 600, ReshHistoryMinSize: 1000, + + SyncConnectorPullPeriodSeconds: 60, } const headerComment = `## @@ -179,6 +189,14 @@ func processAndFillDefaults(configF *configFile) (Config, error) { config.SyncConnectorAddress = configF.SyncConnectorAddress + if configF.SyncConnectorAuthToken != nil { + config.SyncConnectorAuthToken = *configF.SyncConnectorAuthToken + } + + if configF.SyncConnectorPullPeriodSeconds != nil { + config.SyncConnectorPullPeriodSeconds = *configF.SyncConnectorPullPeriodSeconds + } + return config, err } diff --git a/internal/histcli/histcli.go b/internal/histcli/histcli.go index 8069990..b032be5 100644 --- a/internal/histcli/histcli.go +++ b/internal/histcli/histcli.go @@ -2,29 +2,42 @@ package histcli import ( "github.com/curusarn/resh/internal/recordint" + "sync" ) // Histcli is a dump of history preprocessed for resh cli purposes type Histcli struct { // list of records - List []recordint.SearchApp + list []recordint.SearchApp + lock sync.RWMutex } // New Histcli -func New() Histcli { - return Histcli{} +func New() *Histcli { + return &Histcli{} } // AddRecord to the histcli func (h *Histcli) AddRecord(rec *recordint.Indexed) { cli := recordint.NewSearchApp(rec) + h.lock.Lock() + defer h.lock.Unlock() - h.List = append(h.List, cli) + h.list = append(h.list, cli) } // AddCmdLine to the histcli func (h *Histcli) AddCmdLine(cmdline string) { cli := recordint.NewSearchAppFromCmdLine(cmdline) + h.lock.Lock() + defer h.lock.Unlock() - h.List = append(h.List, cli) + h.list = append(h.list, cli) +} + +func (h *Histcli) Dump() []recordint.SearchApp { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.list } diff --git a/internal/histfile/histfile.go b/internal/histfile/histfile.go index 33a2279..9b44fb3 100644 --- a/internal/histfile/histfile.go +++ b/internal/histfile/histfile.go @@ -30,7 +30,7 @@ type Histfile struct { bashCmdLines histlist.Histlist zshCmdLines histlist.Histlist - cliRecords histcli.Histcli + cliRecords *histcli.Histcli rio *recio.RecIO } @@ -39,7 +39,7 @@ type Histfile struct { func New(sugar *zap.SugaredLogger, input chan recordint.Collect, sessionsToDrop chan string, reshHistoryPath string, bashHistoryPath string, zshHistoryPath string, maxInitHistSize int, minInitHistSizeKB int, - signals chan os.Signal, shutdownDone chan string) *Histfile { + signals chan os.Signal, shutdownDone chan string, histCli *histcli.Histcli) *Histfile { rio := recio.New(sugar.With("module", "histfile")) hf := Histfile{ @@ -48,7 +48,7 @@ func New(sugar *zap.SugaredLogger, input chan recordint.Collect, sessionsToDrop historyPath: reshHistoryPath, bashCmdLines: histlist.New(sugar), zshCmdLines: histlist.New(sugar), - cliRecords: histcli.New(), + cliRecords: histCli, rio: &rio, } go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB) @@ -70,7 +70,7 @@ func (h *Histfile) loadCliRecords(recs []recordint.Indexed) { h.cliRecords.AddRecord(&rec) } h.sugar.Infow("Resh history loaded", - "historyRecordsCount", len(h.cliRecords.List), + "historyRecordsCount", len(h.cliRecords.Dump()), ) } @@ -255,12 +255,6 @@ func (h *Histfile) mergeAndWriteRecord(sugar *zap.SugaredLogger, part1 recordint // } // } -// DumpCliRecords returns enriched records -func (h *Histfile) DumpCliRecords() histcli.Histcli { - // don't forget locks in the future - return h.cliRecords -} - func loadCmdLines(sugar *zap.SugaredLogger, recs []recordint.Indexed) histlist.Histlist { hl := histlist.New(sugar) // go from bottom and deduplicate diff --git a/internal/histio/histio.go b/internal/histio/histio.go index d541f9d..b8486de 100644 --- a/internal/histio/histio.go +++ b/internal/histio/histio.go @@ -25,7 +25,7 @@ func New(sugar *zap.SugaredLogger, dataDir, deviceID string) *Histio { sugarHistio := sugar.With(zap.String("component", "histio")) histDir := path.Join(dataDir, "history") currPath := path.Join(histDir, deviceID) - // TODO: file extenstion for the history, yes or no? (.reshjson vs. ) + // TODO: file extension for the history, yes or no? (.reshjson vs. ) // TODO: discover other history files, exclude current diff --git a/internal/histlist/histlist.go b/internal/histlist/histlist.go index 7c80c7f..94376f8 100644 --- a/internal/histlist/histlist.go +++ b/internal/histlist/histlist.go @@ -35,7 +35,7 @@ func Copy(hl Histlist) Histlist { // AddCmdLine to the histlist func (h *Histlist) AddCmdLine(cmdLine string) { - // lenBefore := len(h.List) + // lenBefore := len(h.list) // lookup idx, found := h.LastIndex[cmdLine] if found { diff --git a/internal/searchapp/test.go b/internal/searchapp/test.go index 9a2d284..4ff2d02 100644 --- a/internal/searchapp/test.go +++ b/internal/searchapp/test.go @@ -22,5 +22,5 @@ func LoadHistoryFromFile(sugar *zap.SugaredLogger, historyPath string, numLines rec := recs[i] cliRecords.AddRecord(&rec) } - return msg.CliResponse{Records: cliRecords.List} + return msg.CliResponse{Records: cliRecords.Dump()} } diff --git a/internal/syncconnector/reader.go b/internal/syncconnector/reader.go new file mode 100644 index 0000000..6c0cc9f --- /dev/null +++ b/internal/syncconnector/reader.go @@ -0,0 +1,12 @@ +package syncconnector + +import "github.com/curusarn/resh/internal/record" + +func (sc SyncConnector) getLatestRecord(machineId *string) (map[string]string, error) { + return map[string]string{}, nil +} + +func (sc SyncConnector) downloadRecords(lastRecords map[string]string) ([]record.V1, error) { + var records []record.V1 + return records, nil +} diff --git a/internal/syncconnector/syncconnector.go b/internal/syncconnector/syncconnector.go new file mode 100644 index 0000000..f3ae344 --- /dev/null +++ b/internal/syncconnector/syncconnector.go @@ -0,0 +1,54 @@ +package syncconnector + +import ( + "github.com/curusarn/resh/internal/histcli" + "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" + "net/url" + "time" +) + +type SyncConnector struct { + sugar *zap.SugaredLogger + + address *url.URL + authToken string + + history *histcli.Histcli + + // TODO periodic push (or from the write channel) + // TODO push period +} + +func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodSeconds int, history *histcli.Histcli) (*SyncConnector, error) { + parsedAddress, err := url.Parse(address) + if err != nil { + return nil, err + } + + sc := &SyncConnector{ + sugar: sugar.With(zap.String("component", "syncConnector")), + authToken: authToken, + address: parsedAddress, + history: history, + } + + // TODO: propagate signals + go func(sc *SyncConnector) { + for _ = range time.Tick(time.Second * time.Duration(pullPeriodSeconds)) { + sc.sugar.Infow("checking remote") + + // Add fake record (this will be produced by the sync connector) + sc.history.AddRecord(&recordint.Indexed{ + Rec: record.V1{ + CmdLine: "__fake_test__", + DeviceID: "__test__", + }, + }) + + } + }(sc) + + return sc, nil +} diff --git a/internal/syncconnector/writer.go b/internal/syncconnector/writer.go new file mode 100644 index 0000000..a341b94 --- /dev/null +++ b/internal/syncconnector/writer.go @@ -0,0 +1,49 @@ +package syncconnector + +import ( + "github.com/curusarn/resh/internal/recordint" +) + +func (sc SyncConnector) write(collect chan recordint.Collect) { + //for { + // func() { + // select { + // case rec := <-collect: + // part := "2" + // if rec.Rec.PartOne { + // part = "1" + // } + // sugar := h.sugar.With( + // "recordCmdLine", rec.Rec.CmdLine, + // "recordPart", part, + // "recordShell", rec.Shell, + // ) + // sc.sugar.Debugw("Got record") + // h.sessionsMutex.Lock() + // defer h.sessionsMutex.Unlock() + // + // // allows nested sessions to merge records properly + // mergeID := rec.SessionID + "_" + strconv.Itoa(rec.Shlvl) + // sugar = sc.sugar.With("mergeID", mergeID) + // if rec.Rec.PartOne { + // if _, found := h.sessions[mergeID]; found { + // msg := "Got another first part of the records before merging the previous one - overwriting!" + // if rec.Shell == "zsh" { + // sc.sugar.Warnw(msg) + // } else { + // sc.sugar.Infow(msg + " Unfortunately this is normal in bash, it can't be prevented.") + // } + // } + // h.sessions[mergeID] = rec + // } else { + // if part1, found := h.sessions[mergeID]; found == false { + // sc.sugar.Warnw("Got second part of record and nothing to merge it with - ignoring!") + // } else { + // delete(h.sessions, mergeID) + // go h.mergeAndWriteRecord(sugar, part1, rec) + // } + // } + // } + // }() + //} +} From 0f349315fe9417f9b8acd0c100796c901d73282f Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Wed, 2 Nov 2022 00:30:39 +0100 Subject: [PATCH 29/35] check the remote and add records --- internal/cfg/cfg.go | 6 ++-- internal/syncconnector/reader.go | 43 ++++++++++++++++++++++++- internal/syncconnector/syncconnector.go | 33 +++++++++++++------ 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index e9a47cb..6193320 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -61,9 +61,9 @@ type Config struct { // SyncConnectorAddress used by the daemon to connect to the Sync Connector // examples: - // - localhost:1234 // - http://localhost:1234 - // - 192.168.1.1:1324 + // - http://localhost:1234 + // - http://192.168.1.1:1324 // - https://domain.tld // - https://domain.tld/resh SyncConnectorAddress *string @@ -130,6 +130,8 @@ const headerComment = `## ` +// TODO: Add description for the new options. + func getConfigPath() (string, error) { fname := "resh.toml" xdgDir, found := os.LookupEnv("XDG_CONFIG_HOME") diff --git a/internal/syncconnector/reader.go b/internal/syncconnector/reader.go index 6c0cc9f..f278189 100644 --- a/internal/syncconnector/reader.go +++ b/internal/syncconnector/reader.go @@ -1,6 +1,15 @@ package syncconnector -import "github.com/curusarn/resh/internal/record" +import ( + "bytes" + "encoding/json" + "github.com/curusarn/resh/internal/record" + "io" + "io/ioutil" + "log" + "net/http" + "time" +) func (sc SyncConnector) getLatestRecord(machineId *string) (map[string]string, error) { return map[string]string{}, nil @@ -8,5 +17,37 @@ func (sc SyncConnector) getLatestRecord(machineId *string) (map[string]string, e func (sc SyncConnector) downloadRecords(lastRecords map[string]string) ([]record.V1, error) { var records []record.V1 + + client := http.Client{ + Timeout: 3 * time.Second, + } + + // TODO: create request based on the local last records + responseBody := bytes.NewBuffer([]byte("{}")) + + address := sc.getAddressWithPath(historyEndpoint) + resp, err := client.Post(address, "application/json", responseBody) + if err != nil { + sc.sugar.Errorw("history request failed", "address", address, "err", err) + return nil, err + } + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + sc.sugar.Errorw("reader close failed", "err", err) + } + }(resp.Body) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Fatalln(err) + } + + err = json.Unmarshal(body, &records) + if err != nil { + sc.sugar.Errorw("Unmarshalling failed", "err", err) + return nil, err + } + return records, nil } diff --git a/internal/syncconnector/syncconnector.go b/internal/syncconnector/syncconnector.go index f3ae344..3d55962 100644 --- a/internal/syncconnector/syncconnector.go +++ b/internal/syncconnector/syncconnector.go @@ -2,13 +2,17 @@ package syncconnector import ( "github.com/curusarn/resh/internal/histcli" - "github.com/curusarn/resh/internal/record" "github.com/curusarn/resh/internal/recordint" "go.uber.org/zap" "net/url" + "path" "time" ) +const storeEndpoint = "/store" +const historyEndpoint = "/history" +const latestEndpoint = "/latest" + type SyncConnector struct { sugar *zap.SugaredLogger @@ -37,18 +41,29 @@ func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodS // TODO: propagate signals go func(sc *SyncConnector) { for _ = range time.Tick(time.Second * time.Duration(pullPeriodSeconds)) { - sc.sugar.Infow("checking remote") + sc.sugar.Debug("checking remote") + + recs, err := sc.downloadRecords(map[string]string{}) + if err != nil { + continue + } + + sc.sugar.Debugf("Got %d records", len(recs)) - // Add fake record (this will be produced by the sync connector) - sc.history.AddRecord(&recordint.Indexed{ - Rec: record.V1{ - CmdLine: "__fake_test__", - DeviceID: "__test__", - }, - }) + for _, rec := range recs { + sc.history.AddRecord(&recordint.Indexed{ + Rec: rec, + }) + } } }(sc) return sc, nil } + +func (sc SyncConnector) getAddressWithPath(endpoint string) string { + address := *sc.address + address.Path = path.Join(address.Path, endpoint) + return address.String() +} From 02de0512443f43d46d708122632315b2ee40c522 Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Sun, 13 Nov 2022 23:24:07 +0100 Subject: [PATCH 30/35] fix import path and comment for latest endpoint --- internal/syncconnector/reader.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/syncconnector/reader.go b/internal/syncconnector/reader.go index f278189..03e096e 100644 --- a/internal/syncconnector/reader.go +++ b/internal/syncconnector/reader.go @@ -3,7 +3,7 @@ package syncconnector import ( "bytes" "encoding/json" - "github.com/curusarn/resh/internal/record" + "github.com/curusarn/resh/record" "io" "io/ioutil" "log" @@ -51,3 +51,8 @@ func (sc SyncConnector) downloadRecords(lastRecords map[string]string) ([]record return records, nil } + +func latest() { + //curl localhost:8080/latest -X POST -d '[]' + //curl localhost:8080/latest -X POST -d '["one"]' +} From 2bd9f1e733d7bb31b26b5eb88a917af8970045a7 Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Sun, 13 Nov 2022 23:43:38 +0100 Subject: [PATCH 31/35] fix go.sum --- go.sum | 8 -------- 1 file changed, 8 deletions(-) diff --git a/go.sum b/go.sum index ec1b8bf..341fecd 100644 --- a/go.sum +++ b/go.sum @@ -47,7 +47,6 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/awesome-gocui/gocui v1.0.0 h1:1bf0DAr2JqWNxGFS8Kex4fM/khICjEnCi+a1+NfWy+w= github.com/awesome-gocui/gocui v1.0.0/go.mod h1:UvP3dP6+UsTGl9IuqP36wzz6Lemo90wn5p3tJvZ2OqY= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= @@ -63,7 +62,6 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -209,10 +207,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -241,7 +237,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= @@ -263,7 +258,6 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -597,10 +591,8 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From ca9b9fddf1ade89004c0daf40e28c0a90528d850 Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Sat, 19 Nov 2022 18:05:23 +0100 Subject: [PATCH 32/35] update depreacted ioutil call --- cmd/daemon/dump.go | 8 ++++---- cmd/daemon/record.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/daemon/dump.go b/cmd/daemon/dump.go index c488996..d76db70 100644 --- a/cmd/daemon/dump.go +++ b/cmd/daemon/dump.go @@ -3,7 +3,7 @@ package main import ( "encoding/json" "github.com/curusarn/resh/internal/histcli" - "io/ioutil" + "io" "net/http" "github.com/curusarn/resh/internal/msg" @@ -18,17 +18,17 @@ type dumpHandler struct { func (h *dumpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { sugar := h.sugar.With(zap.String("endpoint", "/dump")) sugar.Debugw("Handling request, reading body ...") - jsn, err := ioutil.ReadAll(r.Body) + jsn, err := io.ReadAll(r.Body) if err != nil { sugar.Errorw("Error reading body", "error", err) return } - sugar.Debugw("Unmarshaling record ...") + sugar.Debugw("Unmarshalling record ...") mess := msg.CliMsg{} err = json.Unmarshal(jsn, &mess) if err != nil { - sugar.Errorw("Error during unmarshaling", + sugar.Errorw("Error during unmarshalling", "error", err, "payload", jsn, ) diff --git a/cmd/daemon/record.go b/cmd/daemon/record.go index 34e9c1c..1684129 100644 --- a/cmd/daemon/record.go +++ b/cmd/daemon/record.go @@ -2,7 +2,7 @@ package main import ( "encoding/json" - "io/ioutil" + "io" "net/http" "github.com/curusarn/resh/internal/recordint" @@ -25,7 +25,7 @@ func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { sugar := h.sugar.With(zap.String("endpoint", "/record")) sugar.Debugw("Handling request, sending response, reading body ...") w.Write([]byte("OK\n")) - jsn, err := ioutil.ReadAll(r.Body) + jsn, err := io.ReadAll(r.Body) // run rest of the handler as goroutine to prevent any hangups go func() { if err != nil { From 9198239825aabb70665469d46c2595b637b32eb1 Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Sat, 19 Nov 2022 18:06:56 +0100 Subject: [PATCH 33/35] fix typo --- cmd/daemon/record.go | 4 ++-- cmd/daemon/session-init.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/daemon/record.go b/cmd/daemon/record.go index 1684129..f6bea7d 100644 --- a/cmd/daemon/record.go +++ b/cmd/daemon/record.go @@ -33,11 +33,11 @@ func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - sugar.Debugw("Unmarshaling record ...") + sugar.Debugw("Unmarshalling record ...") rec := recordint.Collect{} err = json.Unmarshal(jsn, &rec) if err != nil { - sugar.Errorw("Error during unmarshaling", + sugar.Errorw("Error during unmarshalling", "error", err, "payload", jsn, ) diff --git a/cmd/daemon/session-init.go b/cmd/daemon/session-init.go index 9ddd76f..48d722b 100644 --- a/cmd/daemon/session-init.go +++ b/cmd/daemon/session-init.go @@ -27,11 +27,11 @@ func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - sugar.Debugw("Unmarshaling record ...") + sugar.Debugw("Unmarshalling record ...") rec := recordint.SessionInit{} err = json.Unmarshal(jsn, &rec) if err != nil { - sugar.Errorw("Error during unmarshaling", + sugar.Errorw("Error during unmarshalling", "error", err, "payload", jsn, ) From 12f106dcb7eb043d463fa4d48a18f5d541468f27 Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Sat, 19 Nov 2022 23:28:56 +0100 Subject: [PATCH 34/35] support latest result retrieval in the storage --- cmd/cli/main.go | 6 ++-- cmd/control/cmd/version.go | 4 +-- cmd/daemon/run-server.go | 2 +- cmd/daemon/session-init.go | 4 +-- internal/histcli/histcli.go | 42 ++++++++++++++++++++++--- internal/recordint/searchapp.go | 1 + internal/searchapp/test.go | 2 +- internal/syncconnector/reader.go | 20 +++++++++--- internal/syncconnector/syncconnector.go | 2 +- 9 files changed, 63 insertions(+), 20 deletions(-) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 5d588f9..33b450f 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -6,7 +6,7 @@ import ( "errors" "flag" "fmt" - "io/ioutil" + "io" "net/http" "os" "sort" @@ -482,7 +482,7 @@ func (m manager) normalMode(g *gocui.Gui, v *gocui.View) error { helpLineHeight := 1 const helpLine = "HELP: type to search, UP/DOWN or CTRL+P/N to select, RIGHT to edit, ENTER to execute, CTRL+G to abort, CTRL+C/D to quit; " + "FLAGS: G = this git repo, E# = exit status #" - // "TIP: when resh-cli is launched command line is used as initial search query" + // "TIP: when resh-cli is launched command line is used as initial search query" mainViewHeight := maxY - topBoxHeight - statusLineHeight - helpLineHeight m.s.displayedItemsCount = mainViewHeight @@ -589,7 +589,7 @@ func SendCliMsg(out *output.Output, m msg.CliMsg, port string) msg.CliResponse { } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { out.Fatal("Failed read response", err) } diff --git a/cmd/control/cmd/version.go b/cmd/control/cmd/version.go index 9d86f20..d9228c0 100644 --- a/cmd/control/cmd/version.go +++ b/cmd/control/cmd/version.go @@ -3,7 +3,7 @@ package cmd import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "os" "strconv" @@ -69,7 +69,7 @@ func getDaemonStatus(port int) (msg.StatusResponse, error) { return mess, err } defer resp.Body.Close() - jsn, err := ioutil.ReadAll(resp.Body) + jsn, err := io.ReadAll(resp.Body) if err != nil { out.Fatal("Error while reading 'daemon /status' response", err) } diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index fcf1828..af2abae 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -35,7 +35,7 @@ func (s *Server) Run() { shutdown := make(chan string) - history := histcli.New() + history := histcli.New(s.sugar) // histfile histfileRecords := make(chan recordint.Collect) diff --git a/cmd/daemon/session-init.go b/cmd/daemon/session-init.go index 48d722b..5ee20c8 100644 --- a/cmd/daemon/session-init.go +++ b/cmd/daemon/session-init.go @@ -2,7 +2,7 @@ package main import ( "encoding/json" - "io/ioutil" + "io" "net/http" "github.com/curusarn/resh/internal/recordint" @@ -19,7 +19,7 @@ func (h *sessionInitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { sugar.Debugw("Handling request, sending response, reading body ...") w.Write([]byte("OK\n")) // TODO: should we somehow check for errors here? - jsn, err := ioutil.ReadAll(r.Body) + jsn, err := io.ReadAll(r.Body) // run rest of the handler as goroutine to prevent any hangups go func() { if err != nil { diff --git a/internal/histcli/histcli.go b/internal/histcli/histcli.go index b032be5..175eadc 100644 --- a/internal/histcli/histcli.go +++ b/internal/histcli/histcli.go @@ -2,19 +2,27 @@ package histcli import ( "github.com/curusarn/resh/internal/recordint" + "go.uber.org/zap" "sync" ) // Histcli is a dump of history preprocessed for resh cli purposes type Histcli struct { // list of records - list []recordint.SearchApp - lock sync.RWMutex + list []recordint.SearchApp + knownIds map[string]struct{} + lock sync.RWMutex + sugar *zap.SugaredLogger + latest map[string]float64 } // New Histcli -func New() *Histcli { - return &Histcli{} +func New(sugar *zap.SugaredLogger) *Histcli { + return &Histcli{ + sugar: sugar.With(zap.String("component", "histCli")), + knownIds: map[string]struct{}{}, + latest: map[string]float64{}, + } } // AddRecord to the histcli @@ -23,7 +31,13 @@ func (h *Histcli) AddRecord(rec *recordint.Indexed) { h.lock.Lock() defer h.lock.Unlock() - h.list = append(h.list, cli) + if _, ok := h.knownIds[rec.Rec.RecordID]; !ok { + h.knownIds[rec.Rec.RecordID] = struct{}{} + h.list = append(h.list, cli) + h.updateLatestPerDevice(cli) + } else { + h.sugar.Debugw("Record is already present", "id", rec.Rec.RecordID) + } } // AddCmdLine to the histcli @@ -41,3 +55,21 @@ func (h *Histcli) Dump() []recordint.SearchApp { return h.list } + +// updateLatestPerDevice should be called only with write lock because it does not lock on its own. +func (h *Histcli) updateLatestPerDevice(rec recordint.SearchApp) { + if l, ok := h.latest[rec.DeviceID]; ok { + if rec.Time > l { + h.latest[rec.DeviceID] = rec.Time + } + } else { + h.latest[rec.DeviceID] = rec.Time + } +} + +func (h *Histcli) LatestRecordsPerDevice() map[string]float64 { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.latest +} diff --git a/internal/recordint/searchapp.go b/internal/recordint/searchapp.go index 1b5f26d..1147877 100644 --- a/internal/recordint/searchapp.go +++ b/internal/recordint/searchapp.go @@ -43,6 +43,7 @@ func NewSearchApp(r *Indexed) SearchApp { return SearchApp{ IsRaw: false, SessionID: r.Rec.SessionID, + DeviceID: r.Rec.DeviceID, CmdLine: r.Rec.CmdLine, Host: r.Rec.Device, Pwd: r.Rec.Pwd, diff --git a/internal/searchapp/test.go b/internal/searchapp/test.go index 4ff2d02..d89b805 100644 --- a/internal/searchapp/test.go +++ b/internal/searchapp/test.go @@ -17,7 +17,7 @@ func LoadHistoryFromFile(sugar *zap.SugaredLogger, historyPath string, numLines if numLines != 0 && numLines < len(recs) { recs = recs[:numLines] } - cliRecords := histcli.New() + cliRecords := histcli.New(sugar) for i := len(recs) - 1; i >= 0; i-- { rec := recs[i] cliRecords.AddRecord(&rec) diff --git a/internal/syncconnector/reader.go b/internal/syncconnector/reader.go index 03e096e..a724d35 100644 --- a/internal/syncconnector/reader.go +++ b/internal/syncconnector/reader.go @@ -3,9 +3,9 @@ package syncconnector import ( "bytes" "encoding/json" + "fmt" "github.com/curusarn/resh/record" "io" - "io/ioutil" "log" "net/http" "time" @@ -15,15 +15,25 @@ func (sc SyncConnector) getLatestRecord(machineId *string) (map[string]string, e return map[string]string{}, nil } -func (sc SyncConnector) downloadRecords(lastRecords map[string]string) ([]record.V1, error) { +func (sc SyncConnector) downloadRecords(lastRecords map[string]float64) ([]record.V1, error) { var records []record.V1 client := http.Client{ Timeout: 3 * time.Second, } - // TODO: create request based on the local last records - responseBody := bytes.NewBuffer([]byte("{}")) + latestRes := map[string]string{} + for device, t := range lastRecords { + sc.sugar.Debugf("Latest for %s is %f", device, t) + latestRes[device] = fmt.Sprintf("%.4f", t) + } + + latestJson, err := json.Marshal(latestRes) + if err != nil { + sc.sugar.Errorw("converting latest to JSON failed", "err", err) + return nil, err + } + responseBody := bytes.NewBuffer(latestJson) address := sc.getAddressWithPath(historyEndpoint) resp, err := client.Post(address, "application/json", responseBody) @@ -38,7 +48,7 @@ func (sc SyncConnector) downloadRecords(lastRecords map[string]string) ([]record sc.sugar.Errorw("reader close failed", "err", err) } }(resp.Body) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { log.Fatalln(err) } diff --git a/internal/syncconnector/syncconnector.go b/internal/syncconnector/syncconnector.go index 3d55962..917f66e 100644 --- a/internal/syncconnector/syncconnector.go +++ b/internal/syncconnector/syncconnector.go @@ -43,7 +43,7 @@ func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodS for _ = range time.Tick(time.Second * time.Duration(pullPeriodSeconds)) { sc.sugar.Debug("checking remote") - recs, err := sc.downloadRecords(map[string]string{}) + recs, err := sc.downloadRecords(sc.history.LatestRecordsPerDevice()) if err != nil { continue } From 414a42f767509ccfb9657911b630463f9798ad3a Mon Sep 17 00:00:00 2001 From: Vit Listik Date: Mon, 21 Nov 2022 04:05:40 +0100 Subject: [PATCH 35/35] implement synchronization --- cmd/daemon/run-server.go | 2 +- internal/cfg/cfg.go | 11 ++- internal/histcli/histcli.go | 13 ++- internal/histfile/histfile.go | 16 ++-- internal/syncconnector/reader.go | 64 +++++++++++-- internal/syncconnector/syncconnector.go | 20 +++-- internal/syncconnector/writer.go | 114 +++++++++++++++--------- 7 files changed, 173 insertions(+), 67 deletions(-) diff --git a/cmd/daemon/run-server.go b/cmd/daemon/run-server.go index af2abae..eeb080f 100644 --- a/cmd/daemon/run-server.go +++ b/cmd/daemon/run-server.go @@ -84,7 +84,7 @@ func (s *Server) Run() { s.sugar.Infow("", "sync_addr", s.config.SyncConnectorAddress) if s.config.SyncConnectorAddress != nil { - sc, err := syncconnector.New(s.sugar, *s.config.SyncConnectorAddress, s.config.SyncConnectorAuthToken, s.config.SyncConnectorPullPeriodSeconds, history) + sc, err := syncconnector.New(s.sugar, *s.config.SyncConnectorAddress, s.config.SyncConnectorAuthToken, s.config.SyncConnectorPullPeriodSeconds, s.config.SyncConnectorSendPeriodSeconds, history) if err != nil { s.sugar.Errorw("Sync Connector init failed", "error", err) } else { diff --git a/internal/cfg/cfg.go b/internal/cfg/cfg.go index 6193320..25a0a8f 100644 --- a/internal/cfg/cfg.go +++ b/internal/cfg/cfg.go @@ -33,6 +33,7 @@ type configFile struct { SyncConnectorAddress *string SyncConnectorAuthToken *string SyncConnectorPullPeriodSeconds *int + SyncConnectorSendPeriodSeconds *int } // Config returned by this package to be used in the rest of the project @@ -73,6 +74,9 @@ type Config struct { // SyncConnectorPullPeriodSeconds how often should Resh daemon download history from Sync Connector SyncConnectorPullPeriodSeconds int + + // SyncConnectorSendPeriodSeconds how often should Resh daemon send history to the Sync Connector + SyncConnectorSendPeriodSeconds int } // defaults for config @@ -85,7 +89,8 @@ var defaults = Config{ SessionWatchPeriodSeconds: 600, ReshHistoryMinSize: 1000, - SyncConnectorPullPeriodSeconds: 60, + SyncConnectorPullPeriodSeconds: 30, + SyncConnectorSendPeriodSeconds: 30, } const headerComment = `## @@ -199,6 +204,10 @@ func processAndFillDefaults(configF *configFile) (Config, error) { config.SyncConnectorPullPeriodSeconds = *configF.SyncConnectorPullPeriodSeconds } + if configF.SyncConnectorSendPeriodSeconds != nil { + config.SyncConnectorSendPeriodSeconds = *configF.SyncConnectorSendPeriodSeconds + } + return config, err } diff --git a/internal/histcli/histcli.go b/internal/histcli/histcli.go index 175eadc..6f889a8 100644 --- a/internal/histcli/histcli.go +++ b/internal/histcli/histcli.go @@ -2,6 +2,7 @@ package histcli import ( "github.com/curusarn/resh/internal/recordint" + "github.com/curusarn/resh/record" "go.uber.org/zap" "sync" ) @@ -9,7 +10,9 @@ import ( // Histcli is a dump of history preprocessed for resh cli purposes type Histcli struct { // list of records - list []recordint.SearchApp + list []recordint.SearchApp + // TODO It is not optimal to keep both raw and list but is necessary for syncConnector now + raw []record.V1 knownIds map[string]struct{} lock sync.RWMutex sugar *zap.SugaredLogger @@ -34,6 +37,7 @@ func (h *Histcli) AddRecord(rec *recordint.Indexed) { if _, ok := h.knownIds[rec.Rec.RecordID]; !ok { h.knownIds[rec.Rec.RecordID] = struct{}{} h.list = append(h.list, cli) + h.raw = append(h.raw, rec.Rec) h.updateLatestPerDevice(cli) } else { h.sugar.Debugw("Record is already present", "id", rec.Rec.RecordID) @@ -56,6 +60,13 @@ func (h *Histcli) Dump() []recordint.SearchApp { return h.list } +func (h *Histcli) DumpRaw() []record.V1 { + h.lock.RLock() + defer h.lock.RUnlock() + + return h.raw +} + // updateLatestPerDevice should be called only with write lock because it does not lock on its own. func (h *Histcli) updateLatestPerDevice(rec recordint.SearchApp) { if l, ok := h.latest[rec.DeviceID]; ok { diff --git a/internal/histfile/histfile.go b/internal/histfile/histfile.go index 9b44fb3..032a398 100644 --- a/internal/histfile/histfile.go +++ b/internal/histfile/histfile.go @@ -218,15 +218,13 @@ func (h *Histfile) mergeAndWriteRecord(sugar *zap.SugaredLogger, part1 recordint return } - func() { - cmdLine := rec.CmdLine - h.bashCmdLines.AddCmdLine(cmdLine) - h.zshCmdLines.AddCmdLine(cmdLine) - h.cliRecords.AddRecord(&recordint.Indexed{ - // TODO: is this what we want? - Rec: rec, - }) - }() + cmdLine := rec.CmdLine + h.bashCmdLines.AddCmdLine(cmdLine) + h.zshCmdLines.AddCmdLine(cmdLine) + h.cliRecords.AddRecord(&recordint.Indexed{ + // TODO: is this what we want? + Rec: rec, + }) h.rio.AppendToFile(h.historyPath, []record.V1{rec}) } diff --git a/internal/syncconnector/reader.go b/internal/syncconnector/reader.go index a724d35..729a6d7 100644 --- a/internal/syncconnector/reader.go +++ b/internal/syncconnector/reader.go @@ -6,8 +6,8 @@ import ( "fmt" "github.com/curusarn/resh/record" "io" - "log" "net/http" + "strconv" "time" ) @@ -33,10 +33,10 @@ func (sc SyncConnector) downloadRecords(lastRecords map[string]float64) ([]recor sc.sugar.Errorw("converting latest to JSON failed", "err", err) return nil, err } - responseBody := bytes.NewBuffer(latestJson) + reqBody := bytes.NewBuffer(latestJson) address := sc.getAddressWithPath(historyEndpoint) - resp, err := client.Post(address, "application/json", responseBody) + resp, err := client.Post(address, "application/json", reqBody) if err != nil { sc.sugar.Errorw("history request failed", "address", address, "err", err) return nil, err @@ -50,7 +50,7 @@ func (sc SyncConnector) downloadRecords(lastRecords map[string]float64) ([]recor }(resp.Body) body, err := io.ReadAll(resp.Body) if err != nil { - log.Fatalln(err) + sc.sugar.Warnw("reading response body failed", "err", err) } err = json.Unmarshal(body, &records) @@ -62,7 +62,57 @@ func (sc SyncConnector) downloadRecords(lastRecords map[string]float64) ([]recor return records, nil } -func latest() { - //curl localhost:8080/latest -X POST -d '[]' - //curl localhost:8080/latest -X POST -d '["one"]' +func (sc SyncConnector) latest() (map[string]float64, error) { + var knownDevices []string + for deviceId, _ := range sc.history.LatestRecordsPerDevice() { + knownDevices = append(knownDevices, deviceId) + } + + client := http.Client{ + Timeout: 3 * time.Second, + } + + knownJson, err := json.Marshal(knownDevices) + if err != nil { + sc.sugar.Errorw("converting latest to JSON failed", "err", err) + return nil, err + } + reqBody := bytes.NewBuffer(knownJson) + + address := sc.getAddressWithPath(latestEndpoint) + resp, err := client.Post(address, "application/json", reqBody) + if err != nil { + sc.sugar.Errorw("latest request failed", "address", address, "err", err) + return nil, err + } + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + sc.sugar.Errorw("reader close failed", "err", err) + } + }(resp.Body) + body, err := io.ReadAll(resp.Body) + if err != nil { + sc.sugar.Warnw("reading response body failed", "err", err) + } + + latest := map[string]string{} + + err = json.Unmarshal(body, &latest) + if err != nil { + sc.sugar.Errorw("Unmarshalling failed", "err", err) + return nil, err + } + + l := make(map[string]float64, len(latest)) + for deviceId, ts := range latest { + t, err := strconv.ParseFloat(ts, 64) + if err != nil { + return nil, err + } + l[deviceId] = t + } + + return l, nil } diff --git a/internal/syncconnector/syncconnector.go b/internal/syncconnector/syncconnector.go index 917f66e..f1df3e0 100644 --- a/internal/syncconnector/syncconnector.go +++ b/internal/syncconnector/syncconnector.go @@ -20,12 +20,9 @@ type SyncConnector struct { authToken string history *histcli.Histcli - - // TODO periodic push (or from the write channel) - // TODO push period } -func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodSeconds int, history *histcli.Histcli) (*SyncConnector, error) { +func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodSeconds int, sendPeriodSeconds int, history *histcli.Histcli) (*SyncConnector, error) { parsedAddress, err := url.Parse(address) if err != nil { return nil, err @@ -41,7 +38,7 @@ func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodS // TODO: propagate signals go func(sc *SyncConnector) { for _ = range time.Tick(time.Second * time.Duration(pullPeriodSeconds)) { - sc.sugar.Debug("checking remote") + sc.sugar.Debug("checking remote for new records") recs, err := sc.downloadRecords(sc.history.LatestRecordsPerDevice()) if err != nil { @@ -59,6 +56,19 @@ func New(sugar *zap.SugaredLogger, address string, authToken string, pullPeriodS } }(sc) + go func(sc *SyncConnector) { + // wait to properly load all the records + time.Sleep(time.Second * time.Duration(sendPeriodSeconds)) + for _ = range time.Tick(time.Second * time.Duration(sendPeriodSeconds)) { + sc.sugar.Debug("syncing local records to the remote") + + err := sc.write() + if err != nil { + sc.sugar.Warnw("sending records to the remote failed", "err", err) + } + } + }(sc) + return sc, nil } diff --git a/internal/syncconnector/writer.go b/internal/syncconnector/writer.go index a341b94..cd935cc 100644 --- a/internal/syncconnector/writer.go +++ b/internal/syncconnector/writer.go @@ -1,49 +1,77 @@ package syncconnector import ( - "github.com/curusarn/resh/internal/recordint" + "bytes" + "encoding/json" + "github.com/curusarn/resh/record" + "io" + "net/http" + "strconv" + "time" ) -func (sc SyncConnector) write(collect chan recordint.Collect) { - //for { - // func() { - // select { - // case rec := <-collect: - // part := "2" - // if rec.Rec.PartOne { - // part = "1" - // } - // sugar := h.sugar.With( - // "recordCmdLine", rec.Rec.CmdLine, - // "recordPart", part, - // "recordShell", rec.Shell, - // ) - // sc.sugar.Debugw("Got record") - // h.sessionsMutex.Lock() - // defer h.sessionsMutex.Unlock() - // - // // allows nested sessions to merge records properly - // mergeID := rec.SessionID + "_" + strconv.Itoa(rec.Shlvl) - // sugar = sc.sugar.With("mergeID", mergeID) - // if rec.Rec.PartOne { - // if _, found := h.sessions[mergeID]; found { - // msg := "Got another first part of the records before merging the previous one - overwriting!" - // if rec.Shell == "zsh" { - // sc.sugar.Warnw(msg) - // } else { - // sc.sugar.Infow(msg + " Unfortunately this is normal in bash, it can't be prevented.") - // } - // } - // h.sessions[mergeID] = rec - // } else { - // if part1, found := h.sessions[mergeID]; found == false { - // sc.sugar.Warnw("Got second part of record and nothing to merge it with - ignoring!") - // } else { - // delete(h.sessions, mergeID) - // go h.mergeAndWriteRecord(sugar, part1, rec) - // } - // } - // } - // }() - //} +func (sc SyncConnector) write() error { + latestRemote, err := sc.latest() + if err != nil { + return err + } + latestLocal := sc.history.LatestRecordsPerDevice() + remoteIsOlder := false + for deviceId, lastLocal := range latestLocal { + if lastRemote, ok := latestRemote[deviceId]; !ok { + // Unknown deviceId on the remote - add records have to be sent + remoteIsOlder = true + break + } else if lastLocal > lastRemote { + remoteIsOlder = true + break + } + } + if !remoteIsOlder { + sc.sugar.Debug("No need to sync remote, there are no newer local records") + return nil + } + var toSend []record.V1 + for _, r := range sc.history.DumpRaw() { + t, err := strconv.ParseFloat(r.Time, 64) + if err != nil { + sc.sugar.Warnw("Invalid time for record - skipping", "time", r.Time) + continue + } + l, ok := latestRemote[r.DeviceID] + if ok && l >= t { + continue + } + sc.sugar.Infow("record is newer", "new", t, "old", l, "id", r.RecordID, "deviceid", r.DeviceID) + toSend = append(toSend, r) + } + + client := http.Client{ + Timeout: 3 * time.Second, + } + + toSendJson, err := json.Marshal(toSend) + if err != nil { + sc.sugar.Errorw("converting toSend to JSON failed", "err", err) + return err + } + reqBody := bytes.NewBuffer(toSendJson) + + address := sc.getAddressWithPath(storeEndpoint) + resp, err := client.Post(address, "application/json", reqBody) + if err != nil { + sc.sugar.Errorw("store request failed", "address", address, "err", err) + return err + } + + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + sc.sugar.Errorw("reader close failed", "err", err) + } + }(resp.Body) + + sc.sugar.Debugw("store call", "status", resp.Status) + + return nil }