From ff763252f5a438bcfc976390e2c32cb0d2f39190 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Fri, 13 Sep 2019 21:54:51 +0200 Subject: [PATCH 01/13] rename variables to fit golang guidelines --- collect/resh-collect.go | 40 +++++++++++------------ common/resh-common.go | 36 ++++++++++---------- evaluate/resh-evaluate.go | 4 +-- sanitize-history/resh-sanitize-history.go | 2 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/collect/resh-collect.go b/collect/resh-collect.go index b96cc72..e49e918 100644 --- a/collect/resh-collect.go +++ b/collect/resh-collect.go @@ -30,9 +30,9 @@ func main() { usr, _ := user.Current() dir := usr.HomeDir configPath := filepath.Join(dir, "/.config/resh.toml") - reshUuidPath := filepath.Join(dir, "/.resh/resh-uuid") + reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") - machineIdPath := "/etc/machine-id" + machineIDPath := "/etc/machine-id" var config common.Config if _, err := toml.DecodeFile(configPath, &config); err != nil { @@ -48,7 +48,7 @@ func main() { exitCode := flag.Int("exitCode", -1, "exit code") shell := flag.String("shell", "", "actual shell") uname := flag.String("uname", "", "uname") - sessionId := flag.String("sessionId", "", "resh generated session id") + sessionID := flag.String("sessionId", "", "resh generated session id") // posix variables cols := flag.String("cols", "-1", "$COLUMNS") @@ -82,10 +82,10 @@ func main() { timezoneBefore := flag.String("timezoneBefore", "", "") timezoneAfter := flag.String("timezoneAfter", "", "") - osReleaseId := flag.String("osReleaseId", "", "/etc/os-release ID") - osReleaseVersionId := flag.String("osReleaseVersionId", "", + osReleaseID := flag.String("osReleaseId", "", "/etc/os-release ID") + osReleaseVersionID := flag.String("osReleaseVersionId", "", "/etc/os-release ID") - osReleaseIdLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") + osReleaseIDLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") osReleaseName := flag.String("osReleaseName", "", "/etc/os-release ID") osReleasePrettyName := flag.String("osReleasePrettyName", "", "/etc/os-release ID") @@ -161,8 +161,8 @@ func main() { *gitRemote = "" } - if *osReleaseId == "" { - *osReleaseId = "linux" + if *osReleaseID == "" { + *osReleaseID = "linux" } if *osReleaseName == "" { *osReleaseName = "Linux" @@ -177,7 +177,7 @@ func main() { ExitCode: *exitCode, Shell: *shell, Uname: *uname, - SessionId: *sessionId, + SessionID: *sessionID, // posix Cols: *cols, @@ -220,15 +220,15 @@ func main() { GitDir: gitDir, GitRealDir: gitRealDir, GitOriginRemote: *gitRemote, - MachineId: readFileContent(machineIdPath), + MachineID: readFileContent(machineIDPath), - OsReleaseId: *osReleaseId, - OsReleaseVersionId: *osReleaseVersionId, - OsReleaseIdLike: *osReleaseIdLike, + OsReleaseID: *osReleaseID, + OsReleaseVersionID: *osReleaseVersionID, + OsReleaseIDLike: *osReleaseIDLike, OsReleaseName: *osReleaseName, OsReleasePrettyName: *osReleasePrettyName, - ReshUuid: readFileContent(reshUuidPath), + ReshUUID: readFileContent(reshUUIDPath), ReshVersion: Version, ReshRevision: Revision, } @@ -236,13 +236,13 @@ func main() { } func sendRecord(r common.Record, port string) { - recJson, err := json.Marshal(r) + recJSON, err := json.Marshal(r) if err != nil { log.Fatal("send err 1", err) } req, err := http.NewRequest("POST", "http://localhost:"+port+"/record", - bytes.NewBuffer(recJson)) + bytes.NewBuffer(recJSON)) if err != nil { log.Fatal("send err 2", err) } @@ -279,14 +279,14 @@ func getGitDirs(cdup string, exitCode int, pwd string) (string, string) { func getTimezoneOffsetInSeconds(zone string) float64 { // date +%z -> "+0200" - hours_str := zone[:3] - mins_str := zone[3:] - hours, err := strconv.Atoi(hours_str) + hoursStr := zone[:3] + minsStr := zone[3:] + hours, err := strconv.Atoi(hoursStr) if err != nil { log.Println("err while parsing hours in timezone offset:", err) return -1 } - mins, err := strconv.Atoi(mins_str) + mins, err := strconv.Atoi(minsStr) if err != nil { log.Println("err while parsing mins in timezone offset:", err) return -1 diff --git a/common/resh-common.go b/common/resh-common.go index 7e91094..1e3189a 100644 --- a/common/resh-common.go +++ b/common/resh-common.go @@ -14,7 +14,7 @@ type Record struct { ExitCode int `json:"exitCode"` Shell string `json:"shell"` Uname string `json:"uname"` - SessionId string `json:"sessionId"` + SessionID string `json:"sessionId"` // posix Cols string `json:"cols"` @@ -57,15 +57,15 @@ type Record struct { GitDir string `json:"gitDir"` GitRealDir string `json:"gitRealDir"` GitOriginRemote string `json:"gitOriginRemote"` - MachineId string `json:"machineId"` + MachineID string `json:"machineId"` - OsReleaseId string `json:"osReleaseId"` - OsReleaseVersionId string `json:"osReleaseVersionId"` - OsReleaseIdLike string `json:"osReleaseIdLike"` + OsReleaseID string `json:"osReleaseId"` + OsReleaseVersionID string `json:"osReleaseVersionId"` + OsReleaseIDLike string `json:"osReleaseIdLike"` OsReleaseName string `json:"osReleaseName"` OsReleasePrettyName string `json:"osReleasePrettyName"` - ReshUuid string `json:"reshUuid"` + ReshUUID string `json:"reshUuid"` ReshVersion string `json:"reshVersion"` ReshRevision string `json:"reshRevision"` @@ -88,7 +88,7 @@ type FallbackRecord struct { ExitCode int `json:"exitCode"` Shell string `json:"shell"` Uname string `json:"uname"` - SessionId string `json:"sessionId"` + SessionID string `json:"sessionId"` // posix Cols int `json:"cols"` // notice the in type @@ -131,15 +131,15 @@ type FallbackRecord struct { GitDir string `json:"gitDir"` GitRealDir string `json:"gitRealDir"` GitOriginRemote string `json:"gitOriginRemote"` - MachineId string `json:"machineId"` + MachineID string `json:"machineId"` - OsReleaseId string `json:"osReleaseId"` - OsReleaseVersionId string `json:"osReleaseVersionId"` - OsReleaseIdLike string `json:"osReleaseIdLike"` + OsReleaseID string `json:"osReleaseId"` + OsReleaseVersionID string `json:"osReleaseVersionId"` + OsReleaseIDLike string `json:"osReleaseIdLike"` OsReleaseName string `json:"osReleaseName"` OsReleasePrettyName string `json:"osReleasePrettyName"` - ReshUuid string `json:"reshUuid"` + ReshUUID string `json:"reshUuid"` ReshVersion string `json:"reshVersion"` ReshRevision string `json:"reshRevision"` } @@ -152,7 +152,7 @@ func ConvertRecord(r *FallbackRecord) Record { ExitCode: r.ExitCode, Shell: r.Shell, Uname: r.Uname, - SessionId: r.SessionId, + SessionID: r.SessionID, // posix // these two lines are the only reason we are doing this @@ -196,15 +196,15 @@ func ConvertRecord(r *FallbackRecord) Record { GitDir: r.GitDir, GitRealDir: r.GitRealDir, GitOriginRemote: r.GitOriginRemote, - MachineId: r.MachineId, + MachineID: r.MachineID, - OsReleaseId: r.OsReleaseId, - OsReleaseVersionId: r.OsReleaseVersionId, - OsReleaseIdLike: r.OsReleaseIdLike, + OsReleaseID: r.OsReleaseID, + OsReleaseVersionID: r.OsReleaseVersionID, + OsReleaseIDLike: r.OsReleaseIDLike, OsReleaseName: r.OsReleaseName, OsReleasePrettyName: r.OsReleasePrettyName, - ReshUuid: r.ReshUuid, + ReshUUID: r.ReshUUID, ReshVersion: r.ReshVersion, ReshRevision: r.ReshRevision, } diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go index bef0b24..bcde537 100644 --- a/evaluate/resh-evaluate.go +++ b/evaluate/resh-evaluate.go @@ -186,10 +186,10 @@ func (e *evaluator) processRecords() { var nextID uint64 nextID = 0 for k, record := range e.UsersRecords[i].Devices[j].Records { - id, found := sessionIDs[record.SessionId] + id, found := sessionIDs[record.SessionID] if found == false { id = nextID - sessionIDs[record.SessionId] = id + sessionIDs[record.SessionID] = id nextID++ } record.SeqSessionID = id diff --git a/sanitize-history/resh-sanitize-history.go b/sanitize-history/resh-sanitize-history.go index 5ce0581..a69621b 100644 --- a/sanitize-history/resh-sanitize-history.go +++ b/sanitize-history/resh-sanitize-history.go @@ -153,7 +153,7 @@ func (s *sanitizer) sanitizeRecord(record *common.Record) error { // hash the most sensitive info, do not tokenize record.Host = s.hashToken(record.Host) record.Login = s.hashToken(record.Login) - record.MachineId = s.hashToken(record.MachineId) + record.MachineID = s.hashToken(record.MachineID) var err error // this changes git url a bit but I'm still happy with the result From 4df80d4c67d2186383a5ee14762975fed4cce181 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 14 Sep 2019 00:00:33 +0200 Subject: [PATCH 02/13] improve record structure --- collect/resh-collect.go | 114 ++++++++-------- common/resh-common.go | 157 +++++------------------ evaluate/resh-evaluate-plot.py | 9 +- evaluate/resh-evaluate.go | 65 +++++----- evaluate/strategy-directory-sensitive.go | 2 +- evaluate/strategy-dummy.go | 2 +- evaluate/strategy-frequent.go | 2 +- evaluate/strategy-recent.go | 2 +- 8 files changed, 128 insertions(+), 225 deletions(-) diff --git a/collect/resh-collect.go b/collect/resh-collect.go index e49e918..286faf9 100644 --- a/collect/resh-collect.go +++ b/collect/resh-collect.go @@ -172,65 +172,67 @@ func main() { } rec := common.Record{ - // core - CmdLine: *cmdLine, - ExitCode: *exitCode, - Shell: *shell, - Uname: *uname, - SessionID: *sessionID, - // posix Cols: *cols, Lines: *lines, - - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - Pwd: *pwd, - PwdAfter: *pwdAfter, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - RealPwd: realPwd, - RealPwdAfter: realPwdAfter, - Pid: *pid, - SessionPid: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - TimezoneAfter: *timezoneAfter, - - RealtimeBefore: realtimeBefore, - RealtimeAfter: realtimeAfter, - RealtimeBeforeLocal: realtimeBeforeLocal, - RealtimeAfterLocal: realtimeAfterLocal, - - RealtimeDuration: realtimeDuration, - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - GitDir: gitDir, - GitRealDir: gitRealDir, - GitOriginRemote: *gitRemote, - MachineID: readFileContent(machineIDPath), - - OsReleaseID: *osReleaseID, - OsReleaseVersionID: *osReleaseVersionID, - OsReleaseIDLike: *osReleaseIDLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, - - ReshUUID: readFileContent(reshUUIDPath), - ReshVersion: Version, - ReshRevision: Revision, + // core + BaseRecord: common.BaseRecord{ + CmdLine: *cmdLine, + ExitCode: *exitCode, + Shell: *shell, + Uname: *uname, + SessionID: *sessionID, + + // posix + Home: *home, + Lang: *lang, + LcAll: *lcAll, + Login: *login, + // Path: *path, + Pwd: *pwd, + PwdAfter: *pwdAfter, + ShellEnv: *shellEnv, + Term: *term, + + // non-posix + RealPwd: realPwd, + RealPwdAfter: realPwdAfter, + Pid: *pid, + SessionPid: *sessionPid, + Host: *host, + Hosttype: *hosttype, + Ostype: *ostype, + Machtype: *machtype, + Shlvl: *shlvl, + + // before after + TimezoneBefore: *timezoneBefore, + TimezoneAfter: *timezoneAfter, + + RealtimeBefore: realtimeBefore, + RealtimeAfter: realtimeAfter, + RealtimeBeforeLocal: realtimeBeforeLocal, + RealtimeAfterLocal: realtimeAfterLocal, + + RealtimeDuration: realtimeDuration, + RealtimeSinceSessionStart: realtimeSinceSessionStart, + RealtimeSinceBoot: realtimeSinceBoot, + + GitDir: gitDir, + GitRealDir: gitRealDir, + GitOriginRemote: *gitRemote, + MachineID: readFileContent(machineIDPath), + + OsReleaseID: *osReleaseID, + OsReleaseVersionID: *osReleaseVersionID, + OsReleaseIDLike: *osReleaseIDLike, + OsReleaseName: *osReleaseName, + OsReleasePrettyName: *osReleasePrettyName, + + ReshUUID: readFileContent(reshUUIDPath), + ReshVersion: Version, + ReshRevision: Revision, + }, } sendRecord(rec, strconv.Itoa(config.Port)) } diff --git a/common/resh-common.go b/common/resh-common.go index 1e3189a..908cf9b 100644 --- a/common/resh-common.go +++ b/common/resh-common.go @@ -7,8 +7,8 @@ import ( "github.com/mattn/go-shellwords" ) -// Record representing single executed command with its metadata -type Record struct { +// BaseRecord - common base for Record and FallbackRecord +type BaseRecord struct { // core CmdLine string `json:"cmdLine"` ExitCode int `json:"exitCode"` @@ -17,8 +17,6 @@ type Record struct { SessionID string `json:"sessionId"` // posix - Cols string `json:"cols"` - Lines string `json:"lines"` Home string `json:"home"` Lang string `json:"lang"` LcAll string `json:"lcAll"` @@ -70,156 +68,59 @@ type Record struct { ReshRevision string `json:"reshRevision"` // added by sanitizatizer - Sanitized bool `json:"sanitized"` + Sanitized bool `json:"sanitized,omitempty"` CmdLength int `json:"cmdLength,omitempty"` +} + +// Record representing single executed command with its metadata +type Record struct { + BaseRecord + + Cols string `json:"cols"` + Lines string `json:"lines"` +} + +// EnrichedRecord - record enriched with additional data +type EnrichedRecord struct { + Record // enriching fields - added "later" - FirstWord string `json:"firstWord,omitempty"` - Invalid bool `json:"invalid,omitempty"` - SeqSessionID uint64 `json:"seqSessionID,omitempty"` + FirstWord string `json:"firstWord"` + Invalid bool `json:"invalid"` + SeqSessionID uint64 `json:"seqSessionId"` + // SeqSessionID uint64 `json:"seqSessionId,omitempty"` } // FallbackRecord when record is too old and can't be parsed into regular Record type FallbackRecord struct { + BaseRecord // older version of the record where cols and lines are int - // core - CmdLine string `json:"cmdLine"` - ExitCode int `json:"exitCode"` - Shell string `json:"shell"` - Uname string `json:"uname"` - SessionID string `json:"sessionId"` - - // posix - Cols int `json:"cols"` // notice the in type - Lines int `json:"lines"` // notice the in type - Home string `json:"home"` - Lang string `json:"lang"` - LcAll string `json:"lcAll"` - Login string `json:"login"` - //Path string `json:"path"` - Pwd string `json:"pwd"` - PwdAfter string `json:"pwdAfter"` - ShellEnv string `json:"shellEnv"` - Term string `json:"term"` - - // non-posix"` - RealPwd string `json:"realPwd"` - RealPwdAfter string `json:"realPwdAfter"` - Pid int `json:"pid"` - SessionPid int `json:"sessionPid"` - Host string `json:"host"` - Hosttype string `json:"hosttype"` - Ostype string `json:"ostype"` - Machtype string `json:"machtype"` - Shlvl int `json:"shlvl"` - - // before after - TimezoneBefore string `json:"timezoneBefore"` - TimezoneAfter string `json:"timezoneAfter"` - - RealtimeBefore float64 `json:"realtimeBefore"` - RealtimeAfter float64 `json:"realtimeAfter"` - RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` - RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` - - RealtimeDuration float64 `json:"realtimeDuration"` - RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` - RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` - //Logs []string `json: "logs"` - - GitDir string `json:"gitDir"` - GitRealDir string `json:"gitRealDir"` - GitOriginRemote string `json:"gitOriginRemote"` - MachineID string `json:"machineId"` - - OsReleaseID string `json:"osReleaseId"` - OsReleaseVersionID string `json:"osReleaseVersionId"` - OsReleaseIDLike string `json:"osReleaseIdLike"` - OsReleaseName string `json:"osReleaseName"` - OsReleasePrettyName string `json:"osReleasePrettyName"` - - ReshUUID string `json:"reshUuid"` - ReshVersion string `json:"reshVersion"` - ReshRevision string `json:"reshRevision"` + Cols int `json:"cols"` // notice the int type + Lines int `json:"lines"` // notice the int type } // ConvertRecord from FallbackRecord to Record func ConvertRecord(r *FallbackRecord) Record { return Record{ - // core - CmdLine: r.CmdLine, - ExitCode: r.ExitCode, - Shell: r.Shell, - Uname: r.Uname, - SessionID: r.SessionID, - - // posix + BaseRecord: r.BaseRecord, // these two lines are the only reason we are doing this Cols: strconv.Itoa(r.Cols), Lines: strconv.Itoa(r.Lines), - - Home: r.Home, - Lang: r.Lang, - LcAll: r.LcAll, - Login: r.Login, - // Path: r.path, - Pwd: r.Pwd, - PwdAfter: r.PwdAfter, - ShellEnv: r.ShellEnv, - Term: r.Term, - - // non-posix - RealPwd: r.RealPwd, - RealPwdAfter: r.RealPwdAfter, - Pid: r.Pid, - SessionPid: r.SessionPid, - Host: r.Host, - Hosttype: r.Hosttype, - Ostype: r.Ostype, - Machtype: r.Machtype, - Shlvl: r.Shlvl, - - // before after - TimezoneBefore: r.TimezoneBefore, - TimezoneAfter: r.TimezoneAfter, - - RealtimeBefore: r.RealtimeBefore, - RealtimeAfter: r.RealtimeAfter, - RealtimeBeforeLocal: r.RealtimeBeforeLocal, - RealtimeAfterLocal: r.RealtimeAfterLocal, - - RealtimeDuration: r.RealtimeDuration, - RealtimeSinceSessionStart: r.RealtimeSinceSessionStart, - RealtimeSinceBoot: r.RealtimeSinceBoot, - - GitDir: r.GitDir, - GitRealDir: r.GitRealDir, - GitOriginRemote: r.GitOriginRemote, - MachineID: r.MachineID, - - OsReleaseID: r.OsReleaseID, - OsReleaseVersionID: r.OsReleaseVersionID, - OsReleaseIDLike: r.OsReleaseIDLike, - OsReleaseName: r.OsReleaseName, - OsReleasePrettyName: r.OsReleasePrettyName, - - ReshUUID: r.ReshUUID, - ReshVersion: r.ReshVersion, - ReshRevision: r.ReshRevision, } } // Enrich - adds additional fields to the record -func (r *Record) Enrich() { +func (r Record) Enrich() EnrichedRecord { + record := EnrichedRecord{Record: r} // Get command/first word from commandline - r.FirstWord = GetCommandFromCommandLine(r.CmdLine) + record.FirstWord = GetCommandFromCommandLine(r.CmdLine) err := r.Validate() if err != nil { log.Println("Invalid command:", r.CmdLine) - r.Invalid = true + record.Invalid = true } - r.Invalid = false + return record // TODO: Detect and mark simple commands r.Simple } diff --git a/evaluate/resh-evaluate-plot.py b/evaluate/resh-evaluate-plot.py index 45d9322..2b64485 100755 --- a/evaluate/resh-evaluate-plot.py +++ b/evaluate/resh-evaluate-plot.py @@ -22,11 +22,11 @@ DATA_records_by_session = defaultdict(list) for user in data["UsersRecords"]: for device in user["Devices"]: for record in device["Records"]: - if record["invalid"]: + if "invalid" in record and record["invalid"]: continue DATA_records.append(record) - DATA_records_by_session[record["sessionId"]].append(record) + DATA_records_by_session[record["seqSessionId"]].append(record) DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"])) @@ -39,7 +39,6 @@ async_draw = True # for strategy in data["Strategies"]: # print(json.dumps(strategy)) - def zipf(length): return list(map(lambda x: 1/2**x, range(0, length))) @@ -265,8 +264,8 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05): # graphviz sometimes fails - see above try: - graph.view() - # graph.render('/tmp/resh-graphviz-cmdSeq.gv', view=True) + # graph.view() + graph.render('/tmp/resh-graphviz-cmdSeq-{}.gv'.format(x), view=True) break except Exception as e: trace = traceback.format_exc() diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go index bcde537..53ad49a 100644 --- a/evaluate/resh-evaluate.go +++ b/evaluate/resh-evaluate.go @@ -110,7 +110,7 @@ func main() { type strategy interface { GetTitleAndDescription() (string, string) GetCandidates() []string - AddHistoryRecord(record *common.Record) error + AddHistoryRecord(record *common.EnrichedRecord) error ResetHistory() error } @@ -128,7 +128,7 @@ type strategyJSON struct { type deviceRecords struct { Name string - Records []common.Record + Records []common.EnrichedRecord } type userRecords struct { @@ -184,7 +184,7 @@ func (e *evaluator) processRecords() { for j, device := range e.UsersRecords[i].Devices { sessionIDs := map[string]uint64{} var nextID uint64 - nextID = 0 + nextID = 1 // start with 1 because 0 won't get saved to json for k, record := range e.UsersRecords[i].Devices[j].Records { id, found := sessionIDs[record.SessionID] if found == false { @@ -192,7 +192,7 @@ func (e *evaluator) processRecords() { sessionIDs[record.SessionID] = id nextID++ } - record.SeqSessionID = id + e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id // assert if record.Sanitized != e.sanitizedInput { if e.sanitizedInput { @@ -200,9 +200,6 @@ func (e *evaluator) processRecords() { } log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") } - - e.UsersRecords[i].Devices[j].Records[k].Enrich() - // device.Records = append(device.Records, record) } sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool { if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { @@ -217,30 +214,34 @@ func (e *evaluator) processRecords() { func (e *evaluator) evaluate(strategy strategy) error { title, description := strategy.GetTitleAndDescription() strategyData := strategyJSON{Title: title, Description: description} - for _, record := range e.UsersRecords[0].Devices[0].Records { - candidates := strategy.GetCandidates() - - matchFound := false - for i, candidate := range candidates { - // make an option (--calculate-total) to turn this on/off ? - // if i >= e.maxCandidates { - // break - // } - if candidate == record.CmdLine { - match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} - strategyData.Matches = append(strategyData.Matches, match) - matchFound = true - break + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + for _, record := range e.UsersRecords[i].Devices[j].Records { + candidates := strategy.GetCandidates() + + matchFound := false + for i, candidate := range candidates { + // make an option (--calculate-total) to turn this on/off ? + // if i >= e.maxCandidates { + // break + // } + if candidate == record.CmdLine { + match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} + strategyData.Matches = append(strategyData.Matches, match) + matchFound = true + break + } + } + if matchFound == false { + strategyData.Matches = append(strategyData.Matches, matchJSON{}) + } + err := strategy.AddHistoryRecord(&record) + if err != nil { + log.Println("Error while evauating", err) + return err + } } } - if matchFound == false { - strategyData.Matches = append(strategyData.Matches, matchJSON{}) - } - err := strategy.AddHistoryRecord(&record) - if err != nil { - log.Println("Error while evauating", err) - return err - } } e.Strategies = append(e.Strategies, strategyData) return nil @@ -303,14 +304,14 @@ func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath strin return records } -func (e *evaluator) loadHistoryRecords(fname string) []common.Record { +func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord { file, err := os.Open(fname) if err != nil { log.Fatal("Open() resh history file error:", err) } defer file.Close() - var records []common.Record + var records []common.EnrichedRecord scanner := bufio.NewScanner(file) for scanner.Scan() { record := common.Record{} @@ -334,7 +335,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.Record { if record.CmdLength == 0 { log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") } - records = append(records, record) + records = append(records, record.Enrich()) } return records } diff --git a/evaluate/strategy-directory-sensitive.go b/evaluate/strategy-directory-sensitive.go index 0c00bc4..2915fc3 100644 --- a/evaluate/strategy-directory-sensitive.go +++ b/evaluate/strategy-directory-sensitive.go @@ -21,7 +21,7 @@ func (s *strategyDirectorySensitive) GetCandidates() []string { return s.history[s.lastPwd] } -func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) error { +func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.EnrichedRecord) error { // work on history for PWD pwd := record.Pwd // remove previous occurance of record diff --git a/evaluate/strategy-dummy.go b/evaluate/strategy-dummy.go index 28ed8ec..29f28b7 100644 --- a/evaluate/strategy-dummy.go +++ b/evaluate/strategy-dummy.go @@ -14,7 +14,7 @@ func (s *strategyDummy) GetCandidates() []string { return nil } -func (s *strategyDummy) AddHistoryRecord(record *common.Record) error { +func (s *strategyDummy) AddHistoryRecord(record *common.EnrichedRecord) error { s.history = append(s.history, record.CmdLine) return nil } diff --git a/evaluate/strategy-frequent.go b/evaluate/strategy-frequent.go index c41f852..b88ab91 100644 --- a/evaluate/strategy-frequent.go +++ b/evaluate/strategy-frequent.go @@ -36,7 +36,7 @@ func (s *strategyFrequent) GetCandidates() []string { return hist } -func (s *strategyFrequent) AddHistoryRecord(record *common.Record) error { +func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error { s.history[record.CmdLine]++ return nil } diff --git a/evaluate/strategy-recent.go b/evaluate/strategy-recent.go index 7d24d23..2b6ebd0 100644 --- a/evaluate/strategy-recent.go +++ b/evaluate/strategy-recent.go @@ -14,7 +14,7 @@ func (s *strategyRecent) GetCandidates() []string { return s.history } -func (s *strategyRecent) AddHistoryRecord(record *common.Record) error { +func (s *strategyRecent) AddHistoryRecord(record *common.EnrichedRecord) error { // remove previous occurance of record for i, cmd := range s.history { if cmd == record.CmdLine { From 1bc5ef53f10f2d78051303154bb767efd0d71db0 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sat, 14 Sep 2019 01:17:17 +0200 Subject: [PATCH 03/13] minor changes --- evaluate/resh-evaluate-plot.py | 17 +++++++++++------ evaluate/resh-evaluate.go | 1 + 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/evaluate/resh-evaluate-plot.py b/evaluate/resh-evaluate-plot.py index 2b64485..f652ec5 100755 --- a/evaluate/resh-evaluate-plot.py +++ b/evaluate/resh-evaluate-plot.py @@ -162,7 +162,7 @@ def plot_cmdLineVocabularySize_cmdLinesEntered(): # Figure 3.3. Sequential structure of UNIX command usage, from Figure 4 in Hanson et al. (1984). # Ball diameters are proportional to stationary probability. Lines indicate significant dependencies, # solid ones being more probable (p < .0001) and dashed ones less probable (.005 < p < .0001). -def graph_cmdSequences(node_count=33, edge_minValue=0.05): +def graph_cmdSequences(node_count=33, edge_minValue=0.05, view_graph=True): START_CMD = "_start_" cmd_count = defaultdict(int) cmdSeq_count = defaultdict(lambda: defaultdict(int)) @@ -265,7 +265,7 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05): # graphviz sometimes fails - see above try: # graph.view() - graph.render('/tmp/resh-graphviz-cmdSeq-{}.gv'.format(x), view=True) + graph.render('/tmp/resh-graph-command_sequence-nodeCount_{}-edgeMinVal_{}.gv'.format(node_count, edge_minValue), view=view_graph) break except Exception as e: trace = traceback.format_exc() @@ -419,10 +419,6 @@ def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): plt.show() - -# graph_cmdSequences(node_count=33, edge_minValue=0.05) -graph_cmdSequences(node_count=28, edge_minValue=0.06) - plot_cmdLineFrq_rank() plot_cmdFrq_rank() @@ -432,6 +428,15 @@ plot_cmdVocabularySize_cmdLinesEntered() plot_strategies_matches(20) plot_strategies_charsRecalled(20) +graph_cmdSequences(node_count=33, edge_minValue=0.048) + +# graph_cmdSequences(node_count=28, edge_minValue=0.06) + +# for n in range(29, 35): +# for e in range(44, 56, 2): +# e *= 0.001 +# graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False) + if async_draw: plt.show() # be careful and check if labels fit the display \ No newline at end of file diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go index 53ad49a..ce04081 100644 --- a/evaluate/resh-evaluate.go +++ b/evaluate/resh-evaluate.go @@ -241,6 +241,7 @@ func (e *evaluator) evaluate(strategy strategy) error { return err } } + strategy.ResetHistory() } } e.Strategies = append(e.Strategies, strategyData) From ff878a9d7994fa2d4b688a44fde7988186985399 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Mon, 16 Sep 2019 01:09:32 +0200 Subject: [PATCH 04/13] various improvements in evaluation enrich: add 'command' on top of 'firstWord' - first word is not always command strategies: add 'random' strategy, add markov chain based strategies evaluate: add plot with average recalled characters including prefix matches --- common/resh-common.go | 27 +++++-- evaluate/resh-evaluate-plot.py | 86 ++++++++++++++++++++-- evaluate/resh-evaluate.go | 62 ++++++++++++++-- evaluate/strategy-directory-sensitive.go | 4 +- evaluate/strategy-frequent.go | 2 +- evaluate/strategy-markov-chain-cmd.go | 91 ++++++++++++++++++++++++ evaluate/strategy-markov-chain.go | 70 ++++++++++++++++++ evaluate/strategy-random.go | 51 +++++++++++++ go.mod | 3 + go.sum | 6 ++ 10 files changed, 380 insertions(+), 22 deletions(-) create mode 100644 evaluate/strategy-markov-chain-cmd.go create mode 100644 evaluate/strategy-markov-chain.go create mode 100644 evaluate/strategy-random.go diff --git a/common/resh-common.go b/common/resh-common.go index 908cf9b..7f1d4a6 100644 --- a/common/resh-common.go +++ b/common/resh-common.go @@ -3,6 +3,7 @@ package common import ( "log" "strconv" + "strings" "github.com/mattn/go-shellwords" ) @@ -85,6 +86,7 @@ type EnrichedRecord struct { Record // enriching fields - added "later" + Command string `json:"command"` FirstWord string `json:"firstWord"` Invalid bool `json:"invalid"` SeqSessionID uint64 `json:"seqSessionId"` @@ -114,7 +116,7 @@ func ConvertRecord(r *FallbackRecord) Record { func (r Record) Enrich() EnrichedRecord { record := EnrichedRecord{Record: r} // Get command/first word from commandline - record.FirstWord = GetCommandFromCommandLine(r.CmdLine) + record.Command, record.FirstWord = GetCommandAndFirstWord(r.CmdLine) err := r.Validate() if err != nil { log.Println("Invalid command:", r.CmdLine) @@ -129,17 +131,28 @@ func (r *Record) Validate() error { return nil } -// GetCommandFromCommandLine func -func GetCommandFromCommandLine(cmdLine string) string { +// GetCommandAndFirstWord func +func GetCommandAndFirstWord(cmdLine string) (string, string) { args, err := shellwords.Parse(cmdLine) if err != nil { log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") - return "" + return "", "" } - if len(args) > 0 { - return args[0] + if len(args) == 0 { + return "", "" } - return "" + i := 0 + for true { + // commands in shell sometimes look like this `variable=something command argument otherArgument --option` + // to get the command we skip over tokens that contain '=' + if strings.ContainsRune(args[i], '=') && len(args) > i+1 { + i++ + continue + } + return args[i], args[0] + } + log.Fatal("GetCommandAndFirstWord error: this should not happen!") + return "ERROR", "ERROR" } // Config struct diff --git a/evaluate/resh-evaluate-plot.py b/evaluate/resh-evaluate-plot.py index f652ec5..bb4eb3e 100755 --- a/evaluate/resh-evaluate-plot.py +++ b/evaluate/resh-evaluate-plot.py @@ -80,7 +80,7 @@ def plot_cmdLineFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): def plot_cmdFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): cmd_count = defaultdict(int) for record in DATA_records: - cmd = record["firstWord"] + cmd = record["command"] if cmd == "": continue cmd_count[cmd] += 1 @@ -110,7 +110,7 @@ def plot_cmdVocabularySize_cmdLinesEntered(): cmd_vocabulary = set() y_cmd_count = [0] for record in DATA_records: - cmd = record["firstWord"] + cmd = record["command"] if cmd in cmd_vocabulary: # repeat last value y_cmd_count.append(y_cmd_count[-1]) @@ -173,7 +173,7 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05, view_graph=True): cmd_count[START_CMD] += 1 prev_cmd = START_CMD for record in session: - cmd = record["firstWord"] + cmd = record["command"] cmdSeq_count[prev_cmd][cmd] += 1 cmd_count[cmd] += 1 if cmd not in cmd_id: @@ -347,7 +347,6 @@ def plot_strategies_matches(plot_size=50, selected_strategies=[]): plt.show() - def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) plt.title("Average characters recalled at distance") @@ -419,6 +418,81 @@ def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): plt.show() +def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]): + plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) + plt.title("Average characters recalled at distance (including prefix matches)") + plt.ylabel("Average characters recalled (including prefix matches)") + plt.xlabel("Distance") + x_values = range(1, plot_size+1) + legend = [] + saved_charsRecalled_total = None + saved_dataPoint_count = None + for strategy in data["Strategies"]: + strategy_title = strategy["Title"] + # strategy_description = strategy["Description"] + + dataPoint_count = 0 + matches_total = 0 + charsRecalled = [0] * plot_size + charsRecalled_total = 0 + + for multiMatch in strategy["PrefixMatches"]: + dataPoint_count += 1 + + if not multiMatch["Match"]: + continue + matches_total += 1 + + last_charsRecalled = 0 + for match in multiMatch["Entries"]: + + chars = match["CharsRecalled"] + charsIncrease = chars - last_charsRecalled + assert(charsIncrease > 0) + charsRecalled_total += charsIncrease + + dist = match["Distance"] + if dist > plot_size: + continue + + charsRecalled[dist-1] += charsIncrease + last_charsRecalled = chars + + # recent is very simple strategy so we will believe + # that there is no bug in it and we can use it to determine total + if strategy_title == "recent": + saved_charsRecalled_total = charsRecalled_total + saved_dataPoint_count = dataPoint_count + + if len(selected_strategies) and strategy_title not in selected_strategies: + continue + + acc = 0 + charsRecalled_cumulative = [] + for x in charsRecalled: + acc += x + charsRecalled_cumulative.append(acc) + charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) + + plt.plot(x_values, charsRecalled_average, 'o-') + legend.append(strategy_title) + + assert(saved_charsRecalled_total is not None) + assert(saved_dataPoint_count is not None) + max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) + plt.plot(x_values, max_values, 'r-') + legend.append("maximum possible") + + x_ticks = list(range(1, plot_size+1, 2)) + x_labels = x_ticks[:] + plt.xticks(x_ticks, x_labels) + plt.legend(legend, loc="best") + if async_draw: + plt.draw() + else: + plt.show() + + plot_cmdLineFrq_rank() plot_cmdFrq_rank() @@ -427,6 +501,7 @@ plot_cmdVocabularySize_cmdLinesEntered() plot_strategies_matches(20) plot_strategies_charsRecalled(20) +plot_strategies_charsRecalled_prefix(20) graph_cmdSequences(node_count=33, edge_minValue=0.048) @@ -437,6 +512,7 @@ graph_cmdSequences(node_count=33, edge_minValue=0.048) # e *= 0.001 # graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False) +# be careful and check if labels fit the display + if async_draw: plt.show() -# be careful and check if labels fit the display \ No newline at end of file diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go index ce04081..39ddbb6 100644 --- a/evaluate/resh-evaluate.go +++ b/evaluate/resh-evaluate.go @@ -15,6 +15,8 @@ import ( "sort" "github.com/curusarn/resh/common" + "github.com/jpillora/longestcommon" + "github.com/schollz/progressbar" ) // Version from git set during build @@ -24,6 +26,8 @@ var Version string var Revision string func main() { + const maxCandidates = 50 + usr, _ := user.Current() dir := usr.HomeDir historyPath := filepath.Join(dir, ".resh_history.json") @@ -42,6 +46,8 @@ func main() { plottingScript := flag.String("plotting-script", "resh-evaluate-plot.py", "Script to use for plotting") inputDataRoot := flag.String("input-data-root", "", "Input data root, enables batch mode, looks for files matching --input option") + slow := flag.Bool("slow", false, + "Enables stuff that takes a long time (e.g. markov chain strategies).") flag.Parse() @@ -71,7 +77,7 @@ func main() { } } - evaluator := evaluator{sanitizedInput: *sanitizedInput, maxCandidates: 50, BatchMode: batchMode} + evaluator := evaluator{sanitizedInput: *sanitizedInput, maxCandidates: maxCandidates, BatchMode: batchMode} if batchMode { err := evaluator.initBatchMode(*input, *inputDataRoot) if err != nil { @@ -94,8 +100,26 @@ func main() { frequent.init() directory := strategyDirectorySensitive{} directory.init() + random := strategyRandom{candidatesSize: maxCandidates} + random.init() + + markovCmd := strategyMarkovChainCmd{order: 1} + markovCmd.init() + + markovCmd2 := strategyMarkovChainCmd{order: 2} + markovCmd2.init() + + markov := strategyMarkovChain{order: 1} + markov.init() - strategies = append(strategies, &recent, &frequent, &directory) + markov2 := strategyMarkovChain{order: 2} + markov2.init() + + strategies = append(strategies, &recent, &frequent, &directory, &random) + + if *slow { + strategies = append(strategies, &markovCmd2, &markovCmd, &markov2, &markov) + } for _, strat := range strategies { err := evaluator.evaluate(strat) @@ -120,10 +144,21 @@ type matchJSON struct { CharsRecalled int } +type multiMatchItemJSON struct { + Distance int + CharsRecalled int +} + +type multiMatchJSON struct { + Match bool + Entries []multiMatchItemJSON +} + type strategyJSON struct { - Title string - Description string - Matches []matchJSON + Title string + Description string + Matches []matchJSON + PrefixMatches []multiMatchJSON } type deviceRecords struct { @@ -213,35 +248,50 @@ func (e *evaluator) processRecords() { func (e *evaluator) evaluate(strategy strategy) error { title, description := strategy.GetTitleAndDescription() + log.Println("Evaluating strategy:", title, "-", description) strategyData := strategyJSON{Title: title, Description: description} for i := range e.UsersRecords { for j := range e.UsersRecords[i].Devices { + bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) for _, record := range e.UsersRecords[i].Devices[j].Records { candidates := strategy.GetCandidates() matchFound := false + longestPrefixMatchLength := 0 + multiMatch := multiMatchJSON{} for i, candidate := range candidates { // make an option (--calculate-total) to turn this on/off ? // if i >= e.maxCandidates { // break // } + commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine})) + if commonPrefixLength > longestPrefixMatchLength { + longestPrefixMatchLength = commonPrefixLength + prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength} + multiMatch.Match = true + multiMatch.Entries = append(multiMatch.Entries, prefixMatch) + } if candidate == record.CmdLine { match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} - strategyData.Matches = append(strategyData.Matches, match) matchFound = true + strategyData.Matches = append(strategyData.Matches, match) + strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) break } } if matchFound == false { strategyData.Matches = append(strategyData.Matches, matchJSON{}) + strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) } err := strategy.AddHistoryRecord(&record) if err != nil { log.Println("Error while evauating", err) return err } + bar.Add(1) } strategy.ResetHistory() + fmt.Println() } } e.Strategies = append(e.Strategies, strategyData) diff --git a/evaluate/strategy-directory-sensitive.go b/evaluate/strategy-directory-sensitive.go index 2915fc3..f1ae106 100644 --- a/evaluate/strategy-directory-sensitive.go +++ b/evaluate/strategy-directory-sensitive.go @@ -1,8 +1,6 @@ package main -import ( - "github.com/curusarn/resh/common" -) +import "github.com/curusarn/resh/common" type strategyDirectorySensitive struct { history map[string][]string diff --git a/evaluate/strategy-frequent.go b/evaluate/strategy-frequent.go index b88ab91..5480779 100644 --- a/evaluate/strategy-frequent.go +++ b/evaluate/strategy-frequent.go @@ -42,6 +42,6 @@ func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error } func (s *strategyFrequent) ResetHistory() error { - s.history = map[string]int{} + s.init() return nil } diff --git a/evaluate/strategy-markov-chain-cmd.go b/evaluate/strategy-markov-chain-cmd.go new file mode 100644 index 0000000..ec7a230 --- /dev/null +++ b/evaluate/strategy-markov-chain-cmd.go @@ -0,0 +1,91 @@ +package main + +import ( + "sort" + "strconv" + + "github.com/curusarn/resh/common" + "github.com/mb-14/gomarkov" +) + +type strategyMarkovChainCmd struct { + order int + history []strMarkCmdHistoryEntry + historyCmds []string +} + +type strMarkCmdHistoryEntry struct { + cmd string + cmdLine string +} + +type strMarkCmdEntry struct { + cmd string + transProb float64 +} + +func (s *strategyMarkovChainCmd) init() { + s.history = nil + s.historyCmds = nil +} + +func (s *strategyMarkovChainCmd) GetTitleAndDescription() (string, string) { + return "command-based markov chain (order " + strconv.Itoa(s.order) + ")", "Use command-based markov chain to recommend commands" +} + +func (s *strategyMarkovChainCmd) GetCandidates() []string { + if len(s.history) < s.order { + var hist []string + for _, item := range s.history { + hist = append(hist, item.cmdLine) + } + return hist + } + chain := gomarkov.NewChain(s.order) + + chain.Add(s.historyCmds) + + cmdsSet := map[string]bool{} + var entries []strMarkCmdEntry + for _, cmd := range s.historyCmds { + if cmdsSet[cmd] { + continue + } + cmdsSet[cmd] = true + prob, _ := chain.TransitionProbability(cmd, s.historyCmds[len(s.historyCmds)-s.order:]) + entries = append(entries, strMarkCmdEntry{cmd: cmd, transProb: prob}) + } + sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) + var hist []string + histSet := map[string]bool{} + for i := len(s.history) - 1; i >= 0; i-- { + if histSet[s.history[i].cmdLine] { + continue + } + histSet[s.history[i].cmdLine] = true + if s.history[i].cmd == entries[0].cmd { + hist = append(hist, s.history[i].cmdLine) + } + } + // log.Println("################") + // log.Println(s.history[len(s.history)-s.order:]) + // log.Println(" -> ") + // x := math.Min(float64(len(hist)), 3) + // log.Println(entries[:int(x)]) + // x = math.Min(float64(len(hist)), 5) + // log.Println(hist[:int(x)]) + // log.Println("################") + return hist +} + +func (s *strategyMarkovChainCmd) AddHistoryRecord(record *common.EnrichedRecord) error { + s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command}) + s.historyCmds = append(s.historyCmds, record.Command) + // s.historySet[record.CmdLine] = true + return nil +} + +func (s *strategyMarkovChainCmd) ResetHistory() error { + s.init() + return nil +} diff --git a/evaluate/strategy-markov-chain.go b/evaluate/strategy-markov-chain.go new file mode 100644 index 0000000..adabbb4 --- /dev/null +++ b/evaluate/strategy-markov-chain.go @@ -0,0 +1,70 @@ +package main + +import ( + "sort" + "strconv" + + "github.com/curusarn/resh/common" + "github.com/mb-14/gomarkov" +) + +type strategyMarkovChain struct { + order int + history []string +} + +type strMarkEntry struct { + cmdLine string + transProb float64 +} + +func (s *strategyMarkovChain) init() { + s.history = nil +} + +func (s *strategyMarkovChain) GetTitleAndDescription() (string, string) { + return "markov chain (order " + strconv.Itoa(s.order) + ")", "Use markov chain to recommend commands" +} + +func (s *strategyMarkovChain) GetCandidates() []string { + if len(s.history) < s.order { + return s.history + } + chain := gomarkov.NewChain(s.order) + + chain.Add(s.history) + + cmdLinesSet := map[string]bool{} + var entries []strMarkEntry + for _, cmdLine := range s.history { + if cmdLinesSet[cmdLine] { + continue + } + cmdLinesSet[cmdLine] = true + prob, _ := chain.TransitionProbability(cmdLine, s.history[len(s.history)-s.order:]) + entries = append(entries, strMarkEntry{cmdLine: cmdLine, transProb: prob}) + } + sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) + var hist []string + for _, item := range entries { + hist = append(hist, item.cmdLine) + } + // log.Println("################") + // log.Println(s.history[len(s.history)-s.order:]) + // log.Println(" -> ") + // x := math.Min(float64(len(hist)), 5) + // log.Println(hist[:int(x)]) + // log.Println("################") + return hist +} + +func (s *strategyMarkovChain) AddHistoryRecord(record *common.EnrichedRecord) error { + s.history = append(s.history, record.CmdLine) + // s.historySet[record.CmdLine] = true + return nil +} + +func (s *strategyMarkovChain) ResetHistory() error { + s.init() + return nil +} diff --git a/evaluate/strategy-random.go b/evaluate/strategy-random.go new file mode 100644 index 0000000..c4d7b27 --- /dev/null +++ b/evaluate/strategy-random.go @@ -0,0 +1,51 @@ +package main + +import ( + "math/rand" + "time" + + "github.com/curusarn/resh/common" +) + +type strategyRandom struct { + candidatesSize int + history []string + historySet map[string]bool +} + +func (s *strategyRandom) init() { + s.history = nil + s.historySet = map[string]bool{} +} + +func (s *strategyRandom) GetTitleAndDescription() (string, string) { + return "random", "Use random commands" +} + +func (s *strategyRandom) GetCandidates() []string { + seed := time.Now().UnixNano() + rand.Seed(seed) + var candidates []string + candidateSet := map[string]bool{} + for len(candidates) < s.candidatesSize && len(candidates)*2 < len(s.historySet) { + x := rand.Intn(len(s.history)) + candidate := s.history[x] + if candidateSet[candidate] == false { + candidateSet[candidate] = true + candidates = append(candidates, candidate) + continue + } + } + return candidates +} + +func (s *strategyRandom) AddHistoryRecord(record *common.EnrichedRecord) error { + s.history = append([]string{record.CmdLine}, s.history...) + s.historySet[record.CmdLine] = true + return nil +} + +func (s *strategyRandom) ResetHistory() error { + s.init() + return nil +} diff --git a/go.mod b/go.mod index 9c901e1..b13b13d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,10 @@ go 1.12 require ( github.com/BurntSushi/toml v0.3.1 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 github.com/mattn/go-shellwords v1.0.6 + github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7 + github.com/schollz/progressbar v1.0.0 github.com/wcharczuk/go-chart v2.0.1+incompatible github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa golang.org/x/image v0.0.0-20190902063713-cb417be4ba39 // indirect diff --git a/go.sum b/go.sum index 92beac2..b5684fc 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 h1:1dSBUfGlorLAua2CRx0zFN7kQsTpE2DQSmr7rrTNgY8= +github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629/go.mod h1:mb5nS4uRANwOJSZj8rlCWAfAcGi72GGMIXx+xGOjA7M= github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7 h1:VsJjhYhufMGXICLwLYr8mFVMp8/A+YqmagMHnG/BA/4= +github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7/go.mod h1:zQmHoMvvVJb7cxyt1wGT77lqUaeOFXlogOppOr4uHVo= +github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= +github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= github.com/wcharczuk/go-chart v2.0.1+incompatible h1:0pz39ZAycJFF7ju/1mepnk26RLVLBCWz1STcD3doU0A= github.com/wcharczuk/go-chart v2.0.1+incompatible/go.mod h1:PF5tmL4EIx/7Wf+hEkpCqYi5He4u90sw+0+6FhrryuE= github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa h1:rW+Lu6281ed/4XGuVIa4/YebTRNvoUJlfJ44ktEVwZk= From baeb955841b997dad9de506dba0931e99ab89022 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 24 Sep 2019 00:44:06 +0200 Subject: [PATCH 05/13] evaluate: add strategy record distance, misc improvements --- common/resh-common.go | 245 +++++++++++++++++++++++++-- evaluate/resh-evaluate-plot.py | 17 +- evaluate/resh-evaluate.go | 79 +++++++-- evaluate/strategy-record-distance.go | 68 ++++++++ 4 files changed, 375 insertions(+), 34 deletions(-) create mode 100644 evaluate/strategy-record-distance.go diff --git a/common/resh-common.go b/common/resh-common.go index 7f1d4a6..260c4be 100644 --- a/common/resh-common.go +++ b/common/resh-common.go @@ -1,7 +1,10 @@ package common import ( + "encoding/json" + "errors" "log" + "math" "strconv" "strings" @@ -86,10 +89,12 @@ type EnrichedRecord struct { Record // enriching fields - added "later" - Command string `json:"command"` - FirstWord string `json:"firstWord"` - Invalid bool `json:"invalid"` - SeqSessionID uint64 `json:"seqSessionId"` + Command string `json:"command"` + FirstWord string `json:"firstWord"` + Invalid bool `json:"invalid"` + SeqSessionID uint64 `json:"seqSessionId"` + DebugThisRecord bool `json:"debugThisRecord"` + Errors []string `json:"errors"` // SeqSessionID uint64 `json:"seqSessionId,omitempty"` } @@ -112,14 +117,33 @@ func ConvertRecord(r *FallbackRecord) Record { } } +// ToString - returns record the json +func (r EnrichedRecord) ToString() (string, error) { + jsonRec, err := json.Marshal(r) + if err != nil { + return "marshalling error", err + } + return string(jsonRec), nil +} + // Enrich - adds additional fields to the record func (r Record) Enrich() EnrichedRecord { record := EnrichedRecord{Record: r} // Get command/first word from commandline - record.Command, record.FirstWord = GetCommandAndFirstWord(r.CmdLine) - err := r.Validate() + var err error + record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine) if err != nil { - log.Println("Invalid command:", r.CmdLine) + record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error()) + rec, _ := record.ToString() + log.Println("Invalid command:", rec) + record.Invalid = true + return record + } + err = r.Validate() + if err != nil { + record.Errors = append(record.Errors, "Validate error:"+err.Error()) + rec, _ := record.ToString() + log.Println("Invalid command:", rec) record.Invalid = true } return record @@ -128,18 +152,85 @@ func (r Record) Enrich() EnrichedRecord { // Validate - returns error if the record is invalid func (r *Record) Validate() error { + if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 { + return errors.New("There is no Time") + } + if r.RealPwd == "" || r.RealPwdAfter == "" { + return errors.New("There is no Real Pwd") + } + if r.Pwd == "" || r.PwdAfter == "" { + return errors.New("There is no Pwd") + } + + // TimezoneBefore + // TimezoneAfter + + // RealtimeDuration + // RealtimeSinceSessionStart - TODO: add later + // RealtimeSinceBoot - TODO: add later + + // device extras + // Host + // Hosttype + // Ostype + // Machtype + // OsReleaseID + // OsReleaseVersionID + // OsReleaseIDLike + // OsReleaseName + // OsReleasePrettyName + + // session extras + // Term + // Shlvl + + // static info + // Lang + // LcAll + + // meta + // ReshUUID + // ReshVersion + // ReshRevision + + // added by sanitizatizer + // Sanitized + // CmdLength return nil } +// SetCmdLine sets cmdLine and related members +func (r *EnrichedRecord) SetCmdLine(cmdLine string) { + r.CmdLine = cmdLine + r.CmdLength = len(cmdLine) + r.ExitCode = 0 + var err error + r.Command, r.FirstWord, err = GetCommandAndFirstWord(cmdLine) + if err != nil { + r.Errors = append(r.Errors, "GetCommandAndFirstWord error:"+err.Error()) + // log.Println("Invalid command:", r.CmdLine) + r.Invalid = true + } +} + +// SetBeforeToAfter - set "before" members to "after" members +func (r *EnrichedRecord) SetBeforeToAfter() { + r.Pwd = r.PwdAfter + r.RealPwd = r.RealPwdAfter + // r.TimezoneBefore = r.TimezoneAfter + // r.RealtimeBefore = r.RealtimeAfter + // r.RealtimeBeforeLocal = r.RealtimeAfterLocal +} + // GetCommandAndFirstWord func -func GetCommandAndFirstWord(cmdLine string) (string, string) { +func GetCommandAndFirstWord(cmdLine string) (string, string, error) { args, err := shellwords.Parse(cmdLine) if err != nil { log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") - return "", "" + return "", "", err } if len(args) == 0 { - return "", "" + return "", "", nil } i := 0 for true { @@ -149,10 +240,140 @@ func GetCommandAndFirstWord(cmdLine string) (string, string) { i++ continue } - return args[i], args[0] + return args[i], args[0], nil } log.Fatal("GetCommandAndFirstWord error: this should not happen!") - return "ERROR", "ERROR" + return "ERROR", "ERROR", errors.New("this should not happen - contact developer ;)") +} + +// DistParams is used to supply params to EnrichedRecord.DistanceTo() +type DistParams struct { + ExitCode float64 + MachineID float64 + SessionID float64 + Login float64 + Shell float64 + Pwd float64 + RealPwd float64 + Git float64 + Time float64 +} + +// DistanceTo another record +func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { + var dist float64 + dist = 0 + + // lev distance or something? TODO later + // CmdLine + + // exit code + if r.ExitCode != r2.ExitCode { + if r.ExitCode == 0 || r2.ExitCode == 0 { + // one success + one error -> 1 + dist += 1 * p.ExitCode + } else { + // two different errors + dist += 0.5 * p.ExitCode + } + } + + // machine/device + if r.MachineID != r2.MachineID { + dist += 1 * p.MachineID + } + // Uname + + // session + if r.SessionID != r2.SessionID { + dist += 1 * p.SessionID + } + // Pid - add because of nested shells? + // SessionPid + + // user + if r.Login != r2.Login { + dist += 1 * p.Login + } + // Home + + // shell + if r.Shell != r2.Shell { + dist += 1 * p.Shell + } + // ShellEnv + + // pwd + if r.Pwd != r2.Pwd { + // TODO: compare using hierarchy + // TODO: make more important + dist += 1 * p.Pwd + } + if r.RealPwd != r2.RealPwd { + // TODO: -||- + dist += 1 * p.RealPwd + } + // PwdAfter + // RealPwdAfter + + // git + if r.GitDir != r2.GitDir { + dist += 1 * p.Git + } + if r.GitRealDir != r2.GitRealDir { + dist += 1 * p.Git + } + if r.GitOriginRemote != r2.GitOriginRemote { + dist += 1 * p.Git + } + + // time + // this can actually get negative for differences of less than one second which is fine + // distance grows by 1 with every order + distTime := math.Log10(math.Abs(r.RealtimeBefore-r2.RealtimeBefore)) * p.Time + if math.IsNaN(distTime) == false && math.IsInf(distTime, 0) == false { + dist += distTime + } + // RealtimeBeforeLocal + // RealtimeAfter + // RealtimeAfterLocal + + // TimezoneBefore + // TimezoneAfter + + // RealtimeDuration + // RealtimeSinceSessionStart - TODO: add later + // RealtimeSinceBoot - TODO: add later + + // device extras + // Host + // Hosttype + // Ostype + // Machtype + // OsReleaseID + // OsReleaseVersionID + // OsReleaseIDLike + // OsReleaseName + // OsReleasePrettyName + + // session extras + // Term + // Shlvl + + // static info + // Lang + // LcAll + + // meta + // ReshUUID + // ReshVersion + // ReshRevision + + // added by sanitizatizer + // Sanitized + // CmdLength + + return dist } // Config struct diff --git a/evaluate/resh-evaluate-plot.py b/evaluate/resh-evaluate-plot.py index bb4eb3e..7568800 100755 --- a/evaluate/resh-evaluate-plot.py +++ b/evaluate/resh-evaluate-plot.py @@ -9,6 +9,7 @@ import matplotlib.pyplot as plt import matplotlib.path as mpath import numpy as np from graphviz import Digraph +from datetime import datetime PLOT_WIDTH = 10 # inches PLOT_HEIGHT = 7 # inches @@ -274,7 +275,7 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05, view_graph=True): def plot_strategies_matches(plot_size=50, selected_strategies=[]): plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Matches at distance") + plt.title("Matches at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) plt.ylabel('%' + " of matches") plt.xlabel("Distance") legend = [] @@ -349,7 +350,7 @@ def plot_strategies_matches(plot_size=50, selected_strategies=[]): def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance") + plt.title("Average characters recalled at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) plt.ylabel("Average characters recalled") plt.xlabel("Distance") x_values = range(1, plot_size+1) @@ -420,7 +421,7 @@ def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]): plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance (including prefix matches)") + plt.title("Average characters recalled at distance (including prefix matches) <{}>".format(datetime.now().strftime('%H:%M:%S'))) plt.ylabel("Average characters recalled (including prefix matches)") plt.xlabel("Distance") x_values = range(1, plot_size+1) @@ -493,17 +494,17 @@ def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]): plt.show() -plot_cmdLineFrq_rank() -plot_cmdFrq_rank() +# plot_cmdLineFrq_rank() +# plot_cmdFrq_rank() -plot_cmdLineVocabularySize_cmdLinesEntered() -plot_cmdVocabularySize_cmdLinesEntered() +# plot_cmdLineVocabularySize_cmdLinesEntered() +# plot_cmdVocabularySize_cmdLinesEntered() plot_strategies_matches(20) plot_strategies_charsRecalled(20) plot_strategies_charsRecalled_prefix(20) -graph_cmdSequences(node_count=33, edge_minValue=0.048) +# graph_cmdSequences(node_count=33, edge_minValue=0.048) # graph_cmdSequences(node_count=28, edge_minValue=0.06) diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go index 39ddbb6..3fdd45a 100644 --- a/evaluate/resh-evaluate.go +++ b/evaluate/resh-evaluate.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" "log" + "math/rand" "os" "os/exec" "os/user" @@ -48,6 +49,9 @@ func main() { "Input data root, enables batch mode, looks for files matching --input option") slow := flag.Bool("slow", false, "Enables stuff that takes a long time (e.g. markov chain strategies).") + skipFailedCmds := flag.Bool("skip-failed-cmds", false, + "Skips records with non-zero exit status.") + debugRecords := flag.Float64("debug", 0, "Debug records - percentage of records that should be debugged.") flag.Parse() @@ -77,7 +81,8 @@ func main() { } } - evaluator := evaluator{sanitizedInput: *sanitizedInput, maxCandidates: maxCandidates, BatchMode: batchMode} + evaluator := evaluator{sanitizedInput: *sanitizedInput, maxCandidates: maxCandidates, + BatchMode: batchMode, skipFailedCmds: *skipFailedCmds, debugRecords: *debugRecords} if batchMode { err := evaluator.initBatchMode(*input, *inputDataRoot) if err != nil { @@ -95,29 +100,39 @@ func main() { // dummy := strategyDummy{} // strategies = append(strategies, &dummy) - recent := strategyRecent{} + strategies = append(strategies, &strategyRecent{}) + frequent := strategyFrequent{} frequent.init() - directory := strategyDirectorySensitive{} - directory.init() + strategies = append(strategies, &frequent) + random := strategyRandom{candidatesSize: maxCandidates} random.init() + strategies = append(strategies, &random) - markovCmd := strategyMarkovChainCmd{order: 1} - markovCmd.init() + directory := strategyDirectorySensitive{} + directory.init() + strategies = append(strategies, &directory) + + if *slow { + distanceStaticBest := strategyRecordDistance{ + distParams: common.DistParams{SessionID: 1, Pwd: 10, RealPwd: 10, Time: 1}, + label: "10*pwd,10*realpwd,1*session,time", + } + strategies = append(strategies, &distanceStaticBest) - markovCmd2 := strategyMarkovChainCmd{order: 2} - markovCmd2.init() + markovCmd := strategyMarkovChainCmd{order: 1} + markovCmd.init() - markov := strategyMarkovChain{order: 1} - markov.init() + markovCmd2 := strategyMarkovChainCmd{order: 2} + markovCmd2.init() - markov2 := strategyMarkovChain{order: 2} - markov2.init() + markov := strategyMarkovChain{order: 1} + markov.init() - strategies = append(strategies, &recent, &frequent, &directory, &random) + markov2 := strategyMarkovChain{order: 2} + markov2.init() - if *slow { strategies = append(strategies, &markovCmd2, &markovCmd, &markov2, &markov) } @@ -175,6 +190,8 @@ type evaluator struct { sanitizedInput bool BatchMode bool maxCandidates int + skipFailedCmds bool + debugRecords float64 UsersRecords []userRecords Strategies []strategyJSON } @@ -235,6 +252,10 @@ func (e *evaluator) processRecords() { } log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") } + e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id + if e.debugRecords > 0 && rand.Float64() < e.debugRecords { + e.UsersRecords[i].Devices[j].Records[k].DebugThisRecord = true + } } sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool { if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { @@ -253,8 +274,37 @@ func (e *evaluator) evaluate(strategy strategy) error { for i := range e.UsersRecords { for j := range e.UsersRecords[i].Devices { bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) + var prevRecord common.EnrichedRecord for _, record := range e.UsersRecords[i].Devices[j].Records { + if e.skipFailedCmds && record.ExitCode != 0 { + continue + } candidates := strategy.GetCandidates() + if record.DebugThisRecord { + log.Println() + log.Println("===================================================") + log.Println("STRATEGY:", title, "-", description) + log.Println("===================================================") + log.Println("Previous record:") + if prevRecord.RealtimeBefore == 0 { + log.Println("== NIL") + } else { + rec, _ := prevRecord.ToString() + log.Println(rec) + } + log.Println("---------------------------------------------------") + log.Println("Recommendations for:") + rec, _ := record.ToString() + log.Println(rec) + log.Println("---------------------------------------------------") + for i, candidate := range candidates { + if i > 10 { + break + } + log.Println(string(candidate)) + } + log.Println("===================================================") + } matchFound := false longestPrefixMatchLength := 0 @@ -289,6 +339,7 @@ func (e *evaluator) evaluate(strategy strategy) error { return err } bar.Add(1) + prevRecord = record } strategy.ResetHistory() fmt.Println() diff --git a/evaluate/strategy-record-distance.go b/evaluate/strategy-record-distance.go new file mode 100644 index 0000000..07593e0 --- /dev/null +++ b/evaluate/strategy-record-distance.go @@ -0,0 +1,68 @@ +package main + +import ( + "sort" + "strconv" + + "github.com/curusarn/resh/common" +) + +type strategyRecordDistance struct { + history []common.EnrichedRecord + distParams common.DistParams + maxDepth int + label string +} + +type strDistEntry struct { + cmdLine string + distance float64 +} + +func (s *strategyRecordDistance) init() { + s.history = nil +} + +func (s *strategyRecordDistance) GetTitleAndDescription() (string, string) { + return "record distance (depth:" + strconv.Itoa(s.maxDepth) + ";" + s.label + ")", "Use record distance to recommend commands" +} + +func (s *strategyRecordDistance) GetCandidates() []string { + if len(s.history) == 0 { + return nil + } + var prevRecord common.EnrichedRecord + prevRecord = s.history[0] + prevRecord.SetCmdLine("") + prevRecord.SetBeforeToAfter() + var mapItems []strDistEntry + for i, record := range s.history { + if s.maxDepth != 0 && i > s.maxDepth { + break + } + distance := record.DistanceTo(prevRecord, s.distParams) + mapItems = append(mapItems, strDistEntry{record.CmdLine, distance}) + } + sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) + var hist []string + histSet := map[string]bool{} + for _, item := range mapItems { + if histSet[item.cmdLine] { + continue + } + histSet[item.cmdLine] = true + hist = append(hist, item.cmdLine) + } + return hist +} + +func (s *strategyRecordDistance) AddHistoryRecord(record *common.EnrichedRecord) error { + // append record to front + s.history = append([]common.EnrichedRecord{*record}, s.history...) + return nil +} + +func (s *strategyRecordDistance) ResetHistory() error { + s.init() + return nil +} From 8824893b1586d79f8c6a89f7e05663499ddac183 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 24 Sep 2019 01:51:47 +0200 Subject: [PATCH 06/13] evaluate: add dynamic/tf-idf record distance strategy --- evaluate/resh-evaluate-plot.py | 2 +- evaluate/resh-evaluate.go | 20 +++-- evaluate/strategy-dynamic-record-distance.go | 85 ++++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 evaluate/strategy-dynamic-record-distance.go diff --git a/evaluate/resh-evaluate-plot.py b/evaluate/resh-evaluate-plot.py index 7568800..4ffea19 100755 --- a/evaluate/resh-evaluate-plot.py +++ b/evaluate/resh-evaluate-plot.py @@ -502,7 +502,7 @@ def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]): plot_strategies_matches(20) plot_strategies_charsRecalled(20) -plot_strategies_charsRecalled_prefix(20) +# plot_strategies_charsRecalled_prefix(20) # graph_cmdSequences(node_count=33, edge_minValue=0.048) diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go index 3fdd45a..8a7ea08 100644 --- a/evaluate/resh-evaluate.go +++ b/evaluate/resh-evaluate.go @@ -114,12 +114,22 @@ func main() { directory.init() strategies = append(strategies, &directory) + dynamicDist := strategyDynamicRecordDistance{ + maxDepth: 3000, + distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, + label: "10*pwd,10*realpwd,session,time", + } + dynamicDist.init() + strategies = append(strategies, &dynamicDist) + + distanceStaticBest := strategyRecordDistance{ + maxDepth: 3000, + distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, + label: "10*pwd,10*realpwd,session,time", + } + strategies = append(strategies, &distanceStaticBest) + if *slow { - distanceStaticBest := strategyRecordDistance{ - distParams: common.DistParams{SessionID: 1, Pwd: 10, RealPwd: 10, Time: 1}, - label: "10*pwd,10*realpwd,1*session,time", - } - strategies = append(strategies, &distanceStaticBest) markovCmd := strategyMarkovChainCmd{order: 1} markovCmd.init() diff --git a/evaluate/strategy-dynamic-record-distance.go b/evaluate/strategy-dynamic-record-distance.go new file mode 100644 index 0000000..a7ffbe7 --- /dev/null +++ b/evaluate/strategy-dynamic-record-distance.go @@ -0,0 +1,85 @@ +package main + +import ( + "math" + "sort" + "strconv" + + "github.com/curusarn/resh/common" +) + +type strategyDynamicRecordDistance struct { + history []common.EnrichedRecord + distParams common.DistParams + pwdHistogram map[string]int + realPwdHistogram map[string]int + maxDepth int + label string +} + +type strDynDistEntry struct { + cmdLine string + distance float64 +} + +func (s *strategyDynamicRecordDistance) init() { + s.history = nil + s.pwdHistogram = map[string]int{} + s.realPwdHistogram = map[string]int{} +} + +func (s *strategyDynamicRecordDistance) GetTitleAndDescription() (string, string) { + return "dynamic record distance (depth:" + strconv.Itoa(s.maxDepth) + ";" + s.label + ")", "Use TF-IDF record distance to recommend commands" +} + +func (s *strategyDynamicRecordDistance) idf(count int) float64 { + return math.Log(float64(len(s.history)) / float64(count)) +} + +func (s *strategyDynamicRecordDistance) GetCandidates() []string { + if len(s.history) == 0 { + return nil + } + var prevRecord common.EnrichedRecord + prevRecord = s.history[0] + prevRecord.SetCmdLine("") + prevRecord.SetBeforeToAfter() + var mapItems []strDynDistEntry + for i, record := range s.history { + if s.maxDepth != 0 && i > s.maxDepth { + break + } + distParams := common.DistParams{ + Pwd: s.distParams.Pwd * s.idf(s.pwdHistogram[prevRecord.PwdAfter]), + RealPwd: s.distParams.RealPwd * s.idf(s.realPwdHistogram[prevRecord.RealPwdAfter]), + Time: s.distParams.Time, + SessionID: s.distParams.SessionID, + } + distance := record.DistanceTo(prevRecord, distParams) + mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance}) + } + sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) + var hist []string + histSet := map[string]bool{} + for _, item := range mapItems { + if histSet[item.cmdLine] { + continue + } + histSet[item.cmdLine] = true + hist = append(hist, item.cmdLine) + } + return hist +} + +func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *common.EnrichedRecord) error { + // append record to front + s.history = append([]common.EnrichedRecord{*record}, s.history...) + s.pwdHistogram[record.Pwd]++ + s.realPwdHistogram[record.RealPwd]++ + return nil +} + +func (s *strategyDynamicRecordDistance) ResetHistory() error { + s.init() + return nil +} From c229caced982bcd1aff9b102772123b599158cc0 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Tue, 24 Sep 2019 22:49:08 +0200 Subject: [PATCH 07/13] Restructure project to match guidelines --- .gitignore | 5 +-- Makefile | 39 +++++++------------ README.md | 6 ++- version => VERSION | 0 .../resh-collect.go => cmd/collect/main.go | 10 ++--- daemon/resh-daemon.go => cmd/daemon/main.go | 6 +-- .../resh-evaluate.go => cmd/evaluate/main.go | 27 ++++++------- .../evaluate}/strategy-directory-sensitive.go | 4 +- {evaluate => cmd/evaluate}/strategy-dummy.go | 4 +- .../strategy-dynamic-record-distance.go | 14 +++---- .../evaluate}/strategy-frequent.go | 4 +- .../evaluate}/strategy-markov-chain-cmd.go | 4 +- .../evaluate}/strategy-markov-chain.go | 4 +- {evaluate => cmd/evaluate}/strategy-random.go | 4 +- {evaluate => cmd/evaluate}/strategy-recent.go | 4 +- .../evaluate}/strategy-record-distance.go | 12 +++--- .../sanitize/main.go | 10 ++--- config.toml => conf/config.toml | 0 .../sanitizer}/copyright_information.md | 0 .../sanitizer}/whitelist.txt | 0 .../resh-common.go => pkg/records/records.go | 4 +- .../install_helper.sh | 0 rawinstall.sh => scripts/rawinstall.sh | 0 {evaluate => scripts}/resh-evaluate-plot.py | 0 shellrc.sh => scripts/shellrc.sh | 0 uuid.sh => scripts/uuid.sh | 0 26 files changed, 76 insertions(+), 85 deletions(-) rename version => VERSION (100%) rename collect/resh-collect.go => cmd/collect/main.go (98%) rename daemon/resh-daemon.go => cmd/daemon/main.go (97%) rename evaluate/resh-evaluate.go => cmd/evaluate/main.go (95%) rename {evaluate => cmd/evaluate}/strategy-directory-sensitive.go (87%) rename {evaluate => cmd/evaluate}/strategy-dummy.go (74%) rename {evaluate => cmd/evaluate}/strategy-dynamic-record-distance.go (87%) rename {evaluate => cmd/evaluate}/strategy-frequent.go (87%) rename {evaluate => cmd/evaluate}/strategy-markov-chain-cmd.go (94%) rename {evaluate => cmd/evaluate}/strategy-markov-chain.go (92%) rename {evaluate => cmd/evaluate}/strategy-random.go (89%) rename {evaluate => cmd/evaluate}/strategy-recent.go (82%) rename {evaluate => cmd/evaluate}/strategy-record-distance.go (81%) rename sanitize-history/resh-sanitize-history.go => cmd/sanitize/main.go (97%) rename config.toml => conf/config.toml (100%) rename {sanitizer_data => data/sanitizer}/copyright_information.md (100%) rename {sanitizer_data => data/sanitizer}/whitelist.txt (100%) rename common/resh-common.go => pkg/records/records.go (99%) rename install_helper.sh => scripts/install_helper.sh (100%) rename rawinstall.sh => scripts/rawinstall.sh (100%) rename {evaluate => scripts}/resh-evaluate-plot.py (100%) rename shellrc.sh => scripts/shellrc.sh (100%) rename uuid.sh => scripts/uuid.sh (100%) diff --git a/.gitignore b/.gitignore index 602e54d..36f971e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ -resh-collect -resh-daemon -resh-sanitize-history -resh-evaluate +bin/* diff --git a/Makefile b/Makefile index 687ad36..83fbbc0 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ SHELL=/bin/bash -VERSION=$(shell cat version) +VERSION=$(shell cat VERSION) REVISION=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_revision") GOFLAGS=-ldflags "-X main.Version=${VERSION} -X main.Revision=${REVISION}" autoinstall: - ./install_helper.sh + scripts/install_helper.sh sanitize: # @@ -23,8 +23,8 @@ sanitize: # # # Running history sanitization ... - resh-sanitize-history -trim-hashes 0 --output ~/resh_history_sanitized.json - resh-sanitize-history -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json + resh-sanitize -trim-hashes 0 --output ~/resh_history_sanitized.json + resh-sanitize -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json # # # SUCCESS - ALL DONE! @@ -41,8 +41,7 @@ sanitize: # # - -build: submodules resh-collect resh-daemon resh-sanitize-history resh-evaluate +build: submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize rebuild: make clean @@ -51,15 +50,15 @@ rebuild: clean: rm resh-* -install: build submodules/bash-preexec/bash-preexec.sh shellrc.sh config.toml uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config +install: build submodules/bash-preexec/bash-preexec.sh scripts/shellrc.sh conf/config.toml scripts/uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config # Copying files to resh directory ... cp -f submodules/bash-preexec/bash-preexec.sh ~/.bash-preexec.sh - cp -f config.toml ~/.config/resh.toml - cp -f shellrc.sh ~/.resh/shellrc - cp -f uuid.sh ~/.resh/bin/resh-uuid - cp -f resh-* ~/.resh/bin/ - cp -f evaluate/resh-evaluate-plot.py ~/.resh/bin/ - cp -fr sanitizer_data ~/.resh/ + cp -f conf/config.toml ~/.config/resh.toml + cp -f scripts/shellrc.sh ~/.resh/shellrc + cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid + cp -f bin/* ~/.resh/bin/ + cp -f scripts/resh-evaluate-plot.py ~/.resh/bin/ + cp -fr data/sanitizer ~/.resh/ # backward compatibility: We have a new location for resh history file [ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json # Adding resh shellrc to .bashrc ... @@ -107,17 +106,8 @@ uninstall: # Uninstalling ... -rm -rf ~/.resh/ -resh-daemon: daemon/resh-daemon.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< - -resh-collect: collect/resh-collect.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< - -resh-sanitize-history: sanitize-history/resh-sanitize-history.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< - -resh-evaluate: evaluate/resh-evaluate.go evaluate/strategy-*.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< evaluate/strategy-*.go +bin/resh-%: cmd/%/main.go pkg/*/*.go VERSION + go build ${GOFLAGS} -o $@ cmd/$*/*.go $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config: # Creating dirs ... @@ -129,7 +119,6 @@ $(HOME)/.resh/resh-uuid: .PHONY: submodules build install rebuild uninstall clean autoinstall - submodules: | submodules/bash-preexec/bash-preexec.sh @# sets submodule.recurse to true if unset @# sets status.submoduleSummary to true if unset diff --git a/README.md b/README.md index 74f86d3..dc0f620 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ If you are not happy with it you can uninstall it with a single command (`rm -rf The ultimate point of my thesis is to provide a context-based replacement/enhancement for bash and zsh shell history. The idea is to: + - Save each command with metadata (device, directory, git, time, terminal session pid, ... see example below) - Recommend history based on saved metadata - e.g. it will be easier to get to commands specific to the project you are currently working on (based on directory, git repository url, ...) @@ -45,9 +46,11 @@ If you install RESH, please give me some contact info using this form: https://f ## Installation ### Simplest -Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/rawinstall.sh)"` from anywhere. + +Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/scripts/rawinstall.sh)"` from anywhere. ### Simple + 1. Run `git clone https://github.com/curusarn/resh.git && cd resh` 2. Run `make autoinstall` for assisted build & instalation. - OR Run `make install` if you know how to build Golang projects. @@ -59,6 +62,7 @@ If you install RESH, please give me some contact info using this form: https://f Works in `bash` and `zsh`. Tested on: + - Arch - MacOS - Ubuntu (18.04) diff --git a/version b/VERSION similarity index 100% rename from version rename to VERSION diff --git a/collect/resh-collect.go b/cmd/collect/main.go similarity index 98% rename from collect/resh-collect.go rename to cmd/collect/main.go index 286faf9..41d635b 100644 --- a/collect/resh-collect.go +++ b/cmd/collect/main.go @@ -11,7 +11,7 @@ import ( "os" "github.com/BurntSushi/toml" - common "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" // "os/exec" "os/user" @@ -34,7 +34,7 @@ func main() { machineIDPath := "/etc/machine-id" - var config common.Config + var config records.Config if _, err := toml.DecodeFile(configPath, &config); err != nil { log.Fatal("Error reading config:", err) } @@ -171,12 +171,12 @@ func main() { *osReleasePrettyName = "Linux" } - rec := common.Record{ + rec := records.Record{ // posix Cols: *cols, Lines: *lines, // core - BaseRecord: common.BaseRecord{ + BaseRecord: records.BaseRecord{ CmdLine: *cmdLine, ExitCode: *exitCode, Shell: *shell, @@ -237,7 +237,7 @@ func main() { sendRecord(rec, strconv.Itoa(config.Port)) } -func sendRecord(r common.Record, port string) { +func sendRecord(r records.Record, port string) { recJSON, err := json.Marshal(r) if err != nil { log.Fatal("send err 1", err) diff --git a/daemon/resh-daemon.go b/cmd/daemon/main.go similarity index 97% rename from daemon/resh-daemon.go rename to cmd/daemon/main.go index c8bcb0e..e2f7997 100644 --- a/daemon/resh-daemon.go +++ b/cmd/daemon/main.go @@ -14,7 +14,7 @@ import ( "strings" "github.com/BurntSushi/toml" - common "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" ) // Version from git set during build @@ -43,7 +43,7 @@ func main() { log.SetOutput(f) log.SetPrefix(strconv.Itoa(os.Getpid()) + " | ") - var config common.Config + var config records.Config if _, err := toml.DecodeFile(configPath, &config); err != nil { log.Println("Error reading config", err) return @@ -88,7 +88,7 @@ type recordHandler struct { func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Write([]byte("OK\n")) - record := common.Record{} + record := records.Record{} jsn, err := ioutil.ReadAll(r.Body) if err != nil { diff --git a/evaluate/resh-evaluate.go b/cmd/evaluate/main.go similarity index 95% rename from evaluate/resh-evaluate.go rename to cmd/evaluate/main.go index 8a7ea08..0290f59 100644 --- a/evaluate/resh-evaluate.go +++ b/cmd/evaluate/main.go @@ -15,8 +15,9 @@ import ( "path/filepath" "sort" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" "github.com/jpillora/longestcommon" + "github.com/schollz/progressbar" ) @@ -116,7 +117,7 @@ func main() { dynamicDist := strategyDynamicRecordDistance{ maxDepth: 3000, - distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, + distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, label: "10*pwd,10*realpwd,session,time", } dynamicDist.init() @@ -124,7 +125,7 @@ func main() { distanceStaticBest := strategyRecordDistance{ maxDepth: 3000, - distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, + distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, label: "10*pwd,10*realpwd,session,time", } strategies = append(strategies, &distanceStaticBest) @@ -159,7 +160,7 @@ func main() { type strategy interface { GetTitleAndDescription() (string, string) GetCandidates() []string - AddHistoryRecord(record *common.EnrichedRecord) error + AddHistoryRecord(record *records.EnrichedRecord) error ResetHistory() error } @@ -188,7 +189,7 @@ type strategyJSON struct { type deviceRecords struct { Name string - Records []common.EnrichedRecord + Records []records.EnrichedRecord } type userRecords struct { @@ -284,7 +285,7 @@ func (e *evaluator) evaluate(strategy strategy) error { for i := range e.UsersRecords { for j := range e.UsersRecords[i].Devices { bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) - var prevRecord common.EnrichedRecord + var prevRecord records.EnrichedRecord for _, record := range e.UsersRecords[i].Devices[j].Records { if e.skipFailedCmds && record.ExitCode != 0 { continue @@ -416,18 +417,18 @@ func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath strin return records } -func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord { +func (e *evaluator) loadHistoryRecords(fname string) []records.EnrichedRecord { file, err := os.Open(fname) if err != nil { log.Fatal("Open() resh history file error:", err) } defer file.Close() - var records []common.EnrichedRecord + var recs []records.EnrichedRecord scanner := bufio.NewScanner(file) for scanner.Scan() { - record := common.Record{} - fallbackRecord := common.FallbackRecord{} + record := records.Record{} + fallbackRecord := records.FallbackRecord{} line := scanner.Text() err = json.Unmarshal([]byte(line), &record) if err != nil { @@ -436,7 +437,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord { log.Println("Line:", line) log.Fatal("Decoding error:", err) } - record = common.ConvertRecord(&fallbackRecord) + record = records.ConvertRecord(&fallbackRecord) } if e.sanitizedInput == false { if record.CmdLength != 0 { @@ -447,7 +448,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord { if record.CmdLength == 0 { log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") } - records = append(records, record.Enrich()) + recs = append(recs, record.Enrich()) } - return records + return recs } diff --git a/evaluate/strategy-directory-sensitive.go b/cmd/evaluate/strategy-directory-sensitive.go similarity index 87% rename from evaluate/strategy-directory-sensitive.go rename to cmd/evaluate/strategy-directory-sensitive.go index f1ae106..a05deac 100644 --- a/evaluate/strategy-directory-sensitive.go +++ b/cmd/evaluate/strategy-directory-sensitive.go @@ -1,6 +1,6 @@ package main -import "github.com/curusarn/resh/common" +import "github.com/curusarn/resh/pkg/records" type strategyDirectorySensitive struct { history map[string][]string @@ -19,7 +19,7 @@ func (s *strategyDirectorySensitive) GetCandidates() []string { return s.history[s.lastPwd] } -func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyDirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error { // work on history for PWD pwd := record.Pwd // remove previous occurance of record diff --git a/evaluate/strategy-dummy.go b/cmd/evaluate/strategy-dummy.go similarity index 74% rename from evaluate/strategy-dummy.go rename to cmd/evaluate/strategy-dummy.go index 29f28b7..9d20779 100644 --- a/evaluate/strategy-dummy.go +++ b/cmd/evaluate/strategy-dummy.go @@ -1,6 +1,6 @@ package main -import "github.com/curusarn/resh/common" +import "github.com/curusarn/resh/pkg/records" type strategyDummy struct { history []string @@ -14,7 +14,7 @@ func (s *strategyDummy) GetCandidates() []string { return nil } -func (s *strategyDummy) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyDummy) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, record.CmdLine) return nil } diff --git a/evaluate/strategy-dynamic-record-distance.go b/cmd/evaluate/strategy-dynamic-record-distance.go similarity index 87% rename from evaluate/strategy-dynamic-record-distance.go rename to cmd/evaluate/strategy-dynamic-record-distance.go index a7ffbe7..fdaa820 100644 --- a/evaluate/strategy-dynamic-record-distance.go +++ b/cmd/evaluate/strategy-dynamic-record-distance.go @@ -5,12 +5,12 @@ import ( "sort" "strconv" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" ) type strategyDynamicRecordDistance struct { - history []common.EnrichedRecord - distParams common.DistParams + history []records.EnrichedRecord + distParams records.DistParams pwdHistogram map[string]int realPwdHistogram map[string]int maxDepth int @@ -40,7 +40,7 @@ func (s *strategyDynamicRecordDistance) GetCandidates() []string { if len(s.history) == 0 { return nil } - var prevRecord common.EnrichedRecord + var prevRecord records.EnrichedRecord prevRecord = s.history[0] prevRecord.SetCmdLine("") prevRecord.SetBeforeToAfter() @@ -49,7 +49,7 @@ func (s *strategyDynamicRecordDistance) GetCandidates() []string { if s.maxDepth != 0 && i > s.maxDepth { break } - distParams := common.DistParams{ + distParams := records.DistParams{ Pwd: s.distParams.Pwd * s.idf(s.pwdHistogram[prevRecord.PwdAfter]), RealPwd: s.distParams.RealPwd * s.idf(s.realPwdHistogram[prevRecord.RealPwdAfter]), Time: s.distParams.Time, @@ -71,9 +71,9 @@ func (s *strategyDynamicRecordDistance) GetCandidates() []string { return hist } -func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { // append record to front - s.history = append([]common.EnrichedRecord{*record}, s.history...) + s.history = append([]records.EnrichedRecord{*record}, s.history...) s.pwdHistogram[record.Pwd]++ s.realPwdHistogram[record.RealPwd]++ return nil diff --git a/evaluate/strategy-frequent.go b/cmd/evaluate/strategy-frequent.go similarity index 87% rename from evaluate/strategy-frequent.go rename to cmd/evaluate/strategy-frequent.go index 5480779..f081742 100644 --- a/evaluate/strategy-frequent.go +++ b/cmd/evaluate/strategy-frequent.go @@ -3,7 +3,7 @@ package main import ( "sort" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" ) type strategyFrequent struct { @@ -36,7 +36,7 @@ func (s *strategyFrequent) GetCandidates() []string { return hist } -func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyFrequent) AddHistoryRecord(record *records.EnrichedRecord) error { s.history[record.CmdLine]++ return nil } diff --git a/evaluate/strategy-markov-chain-cmd.go b/cmd/evaluate/strategy-markov-chain-cmd.go similarity index 94% rename from evaluate/strategy-markov-chain-cmd.go rename to cmd/evaluate/strategy-markov-chain-cmd.go index ec7a230..7c4ae7f 100644 --- a/evaluate/strategy-markov-chain-cmd.go +++ b/cmd/evaluate/strategy-markov-chain-cmd.go @@ -4,7 +4,7 @@ import ( "sort" "strconv" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" "github.com/mb-14/gomarkov" ) @@ -78,7 +78,7 @@ func (s *strategyMarkovChainCmd) GetCandidates() []string { return hist } -func (s *strategyMarkovChainCmd) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyMarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command}) s.historyCmds = append(s.historyCmds, record.Command) // s.historySet[record.CmdLine] = true diff --git a/evaluate/strategy-markov-chain.go b/cmd/evaluate/strategy-markov-chain.go similarity index 92% rename from evaluate/strategy-markov-chain.go rename to cmd/evaluate/strategy-markov-chain.go index adabbb4..25876b9 100644 --- a/evaluate/strategy-markov-chain.go +++ b/cmd/evaluate/strategy-markov-chain.go @@ -4,7 +4,7 @@ import ( "sort" "strconv" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" "github.com/mb-14/gomarkov" ) @@ -58,7 +58,7 @@ func (s *strategyMarkovChain) GetCandidates() []string { return hist } -func (s *strategyMarkovChain) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyMarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, record.CmdLine) // s.historySet[record.CmdLine] = true return nil diff --git a/evaluate/strategy-random.go b/cmd/evaluate/strategy-random.go similarity index 89% rename from evaluate/strategy-random.go rename to cmd/evaluate/strategy-random.go index c4d7b27..f8a59eb 100644 --- a/evaluate/strategy-random.go +++ b/cmd/evaluate/strategy-random.go @@ -4,7 +4,7 @@ import ( "math/rand" "time" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" ) type strategyRandom struct { @@ -39,7 +39,7 @@ func (s *strategyRandom) GetCandidates() []string { return candidates } -func (s *strategyRandom) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyRandom) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append([]string{record.CmdLine}, s.history...) s.historySet[record.CmdLine] = true return nil diff --git a/evaluate/strategy-recent.go b/cmd/evaluate/strategy-recent.go similarity index 82% rename from evaluate/strategy-recent.go rename to cmd/evaluate/strategy-recent.go index 2b6ebd0..967480c 100644 --- a/evaluate/strategy-recent.go +++ b/cmd/evaluate/strategy-recent.go @@ -1,6 +1,6 @@ package main -import "github.com/curusarn/resh/common" +import "github.com/curusarn/resh/pkg/records" type strategyRecent struct { history []string @@ -14,7 +14,7 @@ func (s *strategyRecent) GetCandidates() []string { return s.history } -func (s *strategyRecent) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyRecent) AddHistoryRecord(record *records.EnrichedRecord) error { // remove previous occurance of record for i, cmd := range s.history { if cmd == record.CmdLine { diff --git a/evaluate/strategy-record-distance.go b/cmd/evaluate/strategy-record-distance.go similarity index 81% rename from evaluate/strategy-record-distance.go rename to cmd/evaluate/strategy-record-distance.go index 07593e0..31ed8e0 100644 --- a/evaluate/strategy-record-distance.go +++ b/cmd/evaluate/strategy-record-distance.go @@ -4,12 +4,12 @@ import ( "sort" "strconv" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" ) type strategyRecordDistance struct { - history []common.EnrichedRecord - distParams common.DistParams + history []records.EnrichedRecord + distParams records.DistParams maxDepth int label string } @@ -31,7 +31,7 @@ func (s *strategyRecordDistance) GetCandidates() []string { if len(s.history) == 0 { return nil } - var prevRecord common.EnrichedRecord + var prevRecord records.EnrichedRecord prevRecord = s.history[0] prevRecord.SetCmdLine("") prevRecord.SetBeforeToAfter() @@ -56,9 +56,9 @@ func (s *strategyRecordDistance) GetCandidates() []string { return hist } -func (s *strategyRecordDistance) AddHistoryRecord(record *common.EnrichedRecord) error { +func (s *strategyRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { // append record to front - s.history = append([]common.EnrichedRecord{*record}, s.history...) + s.history = append([]records.EnrichedRecord{*record}, s.history...) return nil } diff --git a/sanitize-history/resh-sanitize-history.go b/cmd/sanitize/main.go similarity index 97% rename from sanitize-history/resh-sanitize-history.go rename to cmd/sanitize/main.go index a69621b..763f29a 100644 --- a/sanitize-history/resh-sanitize-history.go +++ b/cmd/sanitize/main.go @@ -19,7 +19,7 @@ import ( "strings" "unicode" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" giturls "github.com/whilp/git-urls" ) @@ -79,8 +79,8 @@ func main() { scanner := bufio.NewScanner(inputFile) for scanner.Scan() { - record := common.Record{} - fallbackRecord := common.FallbackRecord{} + record := records.Record{} + fallbackRecord := records.FallbackRecord{} line := scanner.Text() err = json.Unmarshal([]byte(line), &record) if err != nil { @@ -89,7 +89,7 @@ func main() { log.Println("Line:", line) log.Fatal("Decoding error:", err) } - record = common.ConvertRecord(&fallbackRecord) + record = records.ConvertRecord(&fallbackRecord) } err = sanitizer.sanitizeRecord(&record) if err != nil { @@ -139,7 +139,7 @@ func loadData(fname string) map[string]bool { return data } -func (s *sanitizer) sanitizeRecord(record *common.Record) error { +func (s *sanitizer) sanitizeRecord(record *records.Record) error { // hash directories of the paths record.Pwd = s.sanitizePath(record.Pwd) record.RealPwd = s.sanitizePath(record.RealPwd) diff --git a/config.toml b/conf/config.toml similarity index 100% rename from config.toml rename to conf/config.toml diff --git a/sanitizer_data/copyright_information.md b/data/sanitizer/copyright_information.md similarity index 100% rename from sanitizer_data/copyright_information.md rename to data/sanitizer/copyright_information.md diff --git a/sanitizer_data/whitelist.txt b/data/sanitizer/whitelist.txt similarity index 100% rename from sanitizer_data/whitelist.txt rename to data/sanitizer/whitelist.txt diff --git a/common/resh-common.go b/pkg/records/records.go similarity index 99% rename from common/resh-common.go rename to pkg/records/records.go index 260c4be..c63950d 100644 --- a/common/resh-common.go +++ b/pkg/records/records.go @@ -1,4 +1,4 @@ -package common +package records import ( "encoding/json" @@ -246,7 +246,7 @@ func GetCommandAndFirstWord(cmdLine string) (string, string, error) { return "ERROR", "ERROR", errors.New("this should not happen - contact developer ;)") } -// DistParams is used to supply params to EnrichedRecord.DistanceTo() +// DistParams is used to supply params to Enrichedrecords.DistanceTo() type DistParams struct { ExitCode float64 MachineID float64 diff --git a/install_helper.sh b/scripts/install_helper.sh similarity index 100% rename from install_helper.sh rename to scripts/install_helper.sh diff --git a/rawinstall.sh b/scripts/rawinstall.sh similarity index 100% rename from rawinstall.sh rename to scripts/rawinstall.sh diff --git a/evaluate/resh-evaluate-plot.py b/scripts/resh-evaluate-plot.py similarity index 100% rename from evaluate/resh-evaluate-plot.py rename to scripts/resh-evaluate-plot.py diff --git a/shellrc.sh b/scripts/shellrc.sh similarity index 100% rename from shellrc.sh rename to scripts/shellrc.sh diff --git a/uuid.sh b/scripts/uuid.sh similarity index 100% rename from uuid.sh rename to scripts/uuid.sh From 96cf2ae032eaa6ab12ef2facc8fb1bfa446d31c5 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Wed, 25 Sep 2019 02:18:35 +0200 Subject: [PATCH 08/13] split cfg from records, add bash-like strategy, improvements --- cmd/collect/main.go | 3 +- cmd/daemon/main.go | 3 +- cmd/evaluate/main.go | 167 ++++++++++++------ .../strategy-dynamic-record-distance.go | 28 +-- cmd/evaluate/strategy-recent-bash.go | 50 ++++++ cmd/evaluate/strategy-record-distance.go | 8 +- pkg/cfg/cfg.go | 6 + pkg/records/records.go | 34 ++-- 8 files changed, 211 insertions(+), 88 deletions(-) create mode 100644 cmd/evaluate/strategy-recent-bash.go create mode 100644 pkg/cfg/cfg.go diff --git a/cmd/collect/main.go b/cmd/collect/main.go index 41d635b..75cc61d 100644 --- a/cmd/collect/main.go +++ b/cmd/collect/main.go @@ -11,6 +11,7 @@ import ( "os" "github.com/BurntSushi/toml" + "github.com/curusarn/resh/pkg/cfg" "github.com/curusarn/resh/pkg/records" // "os/exec" @@ -34,7 +35,7 @@ func main() { machineIDPath := "/etc/machine-id" - var config records.Config + var config cfg.Config if _, err := toml.DecodeFile(configPath, &config); err != nil { log.Fatal("Error reading config:", err) } diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index e2f7997..9411350 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/curusarn/resh/pkg/cfg" "github.com/curusarn/resh/pkg/records" ) @@ -43,7 +44,7 @@ func main() { log.SetOutput(f) log.SetPrefix(strconv.Itoa(os.Getpid()) + " | ") - var config records.Config + var config cfg.Config if _, err := toml.DecodeFile(configPath, &config); err != nil { log.Println("Error reading config", err) return diff --git a/cmd/evaluate/main.go b/cmd/evaluate/main.go index 0290f59..6e013cc 100644 --- a/cmd/evaluate/main.go +++ b/cmd/evaluate/main.go @@ -13,7 +13,6 @@ import ( "os/exec" "os/user" "path/filepath" - "sort" "github.com/curusarn/resh/pkg/records" "github.com/jpillora/longestcommon" @@ -49,7 +48,7 @@ func main() { inputDataRoot := flag.String("input-data-root", "", "Input data root, enables batch mode, looks for files matching --input option") slow := flag.Bool("slow", false, - "Enables stuff that takes a long time (e.g. markov chain strategies).") + "Enables strategies that takes a long time (e.g. markov chain strategies).") skipFailedCmds := flag.Bool("skip-failed-cmds", false, "Skips records with non-zero exit status.") debugRecords := flag.Float64("debug", 0, "Debug records - percentage of records that should be debugged.") @@ -96,32 +95,33 @@ func main() { } } - var strategies []strategy + var simpleStrategies []ISimpleStrategy + var strategies []IStrategy // dummy := strategyDummy{} - // strategies = append(strategies, &dummy) + // simpleStrategies = append(simpleStrategies, &dummy) - strategies = append(strategies, &strategyRecent{}) + simpleStrategies = append(simpleStrategies, &strategyRecent{}) - frequent := strategyFrequent{} - frequent.init() - strategies = append(strategies, &frequent) + // frequent := strategyFrequent{} + // frequent.init() + // simpleStrategies = append(simpleStrategies, &frequent) - random := strategyRandom{candidatesSize: maxCandidates} - random.init() - strategies = append(strategies, &random) + // random := strategyRandom{candidatesSize: maxCandidates} + // random.init() + // simpleStrategies = append(simpleStrategies, &random) directory := strategyDirectorySensitive{} directory.init() - strategies = append(strategies, &directory) + simpleStrategies = append(simpleStrategies, &directory) - dynamicDist := strategyDynamicRecordDistance{ + dynamicDistG := strategyDynamicRecordDistance{ maxDepth: 3000, - distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, - label: "10*pwd,10*realpwd,session,time", + distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1, Git: 10}, + label: "10*pwd,10*realpwd,session,time,10*git", } - dynamicDist.init() - strategies = append(strategies, &dynamicDist) + dynamicDistG.init() + strategies = append(strategies, &dynamicDistG) distanceStaticBest := strategyRecordDistance{ maxDepth: 3000, @@ -130,6 +130,10 @@ func main() { } strategies = append(strategies, &distanceStaticBest) + recentBash := strategyRecentBash{} + recentBash.init() + strategies = append(strategies, &recentBash) + if *slow { markovCmd := strategyMarkovChainCmd{order: 1} @@ -144,7 +148,11 @@ func main() { markov2 := strategyMarkovChain{order: 2} markov2.init() - strategies = append(strategies, &markovCmd2, &markovCmd, &markov2, &markov) + simpleStrategies = append(simpleStrategies, &markovCmd2, &markovCmd, &markov2, &markov) + } + + for _, strat := range simpleStrategies { + strategies = append(strategies, NewSimpleStrategyWrapper(strat)) } for _, strat := range strategies { @@ -157,13 +165,45 @@ func main() { evaluator.calculateStatsAndPlot(*plottingScript) } -type strategy interface { +type ISimpleStrategy interface { GetTitleAndDescription() (string, string) GetCandidates() []string AddHistoryRecord(record *records.EnrichedRecord) error ResetHistory() error } +type IStrategy interface { + GetTitleAndDescription() (string, string) + GetCandidates(r records.EnrichedRecord) []string + AddHistoryRecord(record *records.EnrichedRecord) error + ResetHistory() error +} + +type simpleStrategyWrapper struct { + strategy ISimpleStrategy +} + +// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy +func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper { + return &simpleStrategyWrapper{strategy: strategy} +} + +func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) { + return s.strategy.GetTitleAndDescription() +} + +func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string { + return s.strategy.GetCandidates() +} + +func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error { + return s.strategy.AddHistoryRecord(r) +} + +func (s *simpleStrategyWrapper) ResetHistory() error { + return s.strategy.ResetHistory() +} + type matchJSON struct { Match bool Distance int @@ -209,7 +249,7 @@ type evaluator struct { func (e *evaluator) initBatchMode(input string, inputDataRoot string) error { e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) - e.processRecords() + e.preprocessRecords() return nil } @@ -219,7 +259,7 @@ func (e *evaluator) init(inputPath string) error { user := userRecords{} user.Devices = append(user.Devices, device) e.UsersRecords = append(e.UsersRecords, user) - e.processRecords() + e.preprocessRecords() return nil } @@ -241,44 +281,61 @@ func (e *evaluator) calculateStatsAndPlot(scriptName string) { } } -// enrich records and add them to serializable structure -func (e *evaluator) processRecords() { - for i := range e.UsersRecords { - for j, device := range e.UsersRecords[i].Devices { - sessionIDs := map[string]uint64{} - var nextID uint64 - nextID = 1 // start with 1 because 0 won't get saved to json - for k, record := range e.UsersRecords[i].Devices[j].Records { - id, found := sessionIDs[record.SessionID] - if found == false { - id = nextID - sessionIDs[record.SessionID] = id - nextID++ - } - e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id - // assert - if record.Sanitized != e.sanitizedInput { - if e.sanitizedInput { - log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") - } - log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") - } - e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id - if e.debugRecords > 0 && rand.Float64() < e.debugRecords { - e.UsersRecords[i].Devices[j].Records[k].DebugThisRecord = true - } +func (e *evaluator) preprocessDeviceRecords(device deviceRecords) deviceRecords { + sessionIDs := map[string]uint64{} + var nextID uint64 + nextID = 1 // start with 1 because 0 won't get saved to json + for k, record := range device.Records { + id, found := sessionIDs[record.SessionID] + if found == false { + id = nextID + sessionIDs[record.SessionID] = id + nextID++ + } + device.Records[k].SeqSessionID = id + // assert + if record.Sanitized != e.sanitizedInput { + if e.sanitizedInput { + log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") } - sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool { - if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { - return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal - } - return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID - }) + log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") + } + device.Records[k].SeqSessionID = id + if e.debugRecords > 0 && rand.Float64() < e.debugRecords { + device.Records[k].DebugThisRecord = true + } + } + // sort.SliceStable(device.Records, func(x, y int) bool { + // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { + // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal + // } + // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID + // }) + + // iterate from back and mark last record of each session + sessionIDSet := map[string]bool{} + for i := len(device.Records) - 1; i >= 0; i-- { + var record *records.EnrichedRecord + record = &device.Records[i] + if sessionIDSet[record.SessionID] { + continue + } + sessionIDSet[record.SessionID] = true + record.LastRecordOfSession = true + } + return device +} + +// enrich records and add sequential session ID +func (e *evaluator) preprocessRecords() { + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) } } } -func (e *evaluator) evaluate(strategy strategy) error { +func (e *evaluator) evaluate(strategy IStrategy) error { title, description := strategy.GetTitleAndDescription() log.Println("Evaluating strategy:", title, "-", description) strategyData := strategyJSON{Title: title, Description: description} @@ -290,7 +347,7 @@ func (e *evaluator) evaluate(strategy strategy) error { if e.skipFailedCmds && record.ExitCode != 0 { continue } - candidates := strategy.GetCandidates() + candidates := strategy.GetCandidates(records.Stripped(record)) if record.DebugThisRecord { log.Println() log.Println("===================================================") diff --git a/cmd/evaluate/strategy-dynamic-record-distance.go b/cmd/evaluate/strategy-dynamic-record-distance.go index fdaa820..0a107e9 100644 --- a/cmd/evaluate/strategy-dynamic-record-distance.go +++ b/cmd/evaluate/strategy-dynamic-record-distance.go @@ -9,12 +9,13 @@ import ( ) type strategyDynamicRecordDistance struct { - history []records.EnrichedRecord - distParams records.DistParams - pwdHistogram map[string]int - realPwdHistogram map[string]int - maxDepth int - label string + history []records.EnrichedRecord + distParams records.DistParams + pwdHistogram map[string]int + realPwdHistogram map[string]int + gitOriginHistogram map[string]int + maxDepth int + label string } type strDynDistEntry struct { @@ -26,6 +27,7 @@ func (s *strategyDynamicRecordDistance) init() { s.history = nil s.pwdHistogram = map[string]int{} s.realPwdHistogram = map[string]int{} + s.gitOriginHistogram = map[string]int{} } func (s *strategyDynamicRecordDistance) GetTitleAndDescription() (string, string) { @@ -36,26 +38,23 @@ func (s *strategyDynamicRecordDistance) idf(count int) float64 { return math.Log(float64(len(s.history)) / float64(count)) } -func (s *strategyDynamicRecordDistance) GetCandidates() []string { +func (s *strategyDynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { if len(s.history) == 0 { return nil } - var prevRecord records.EnrichedRecord - prevRecord = s.history[0] - prevRecord.SetCmdLine("") - prevRecord.SetBeforeToAfter() var mapItems []strDynDistEntry for i, record := range s.history { if s.maxDepth != 0 && i > s.maxDepth { break } distParams := records.DistParams{ - Pwd: s.distParams.Pwd * s.idf(s.pwdHistogram[prevRecord.PwdAfter]), - RealPwd: s.distParams.RealPwd * s.idf(s.realPwdHistogram[prevRecord.RealPwdAfter]), + Pwd: s.distParams.Pwd * s.idf(s.pwdHistogram[strippedRecord.PwdAfter]), + RealPwd: s.distParams.RealPwd * s.idf(s.realPwdHistogram[strippedRecord.RealPwdAfter]), + Git: s.distParams.Git * s.idf(s.gitOriginHistogram[strippedRecord.GitOriginRemote]), Time: s.distParams.Time, SessionID: s.distParams.SessionID, } - distance := record.DistanceTo(prevRecord, distParams) + distance := record.DistanceTo(strippedRecord, distParams) mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance}) } sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) @@ -76,6 +75,7 @@ func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *records.Enriche s.history = append([]records.EnrichedRecord{*record}, s.history...) s.pwdHistogram[record.Pwd]++ s.realPwdHistogram[record.RealPwd]++ + s.gitOriginHistogram[record.GitOriginRemote]++ return nil } diff --git a/cmd/evaluate/strategy-recent-bash.go b/cmd/evaluate/strategy-recent-bash.go new file mode 100644 index 0000000..7a83632 --- /dev/null +++ b/cmd/evaluate/strategy-recent-bash.go @@ -0,0 +1,50 @@ +package main + +import "github.com/curusarn/resh/pkg/records" + +type strategyRecentBash struct { + histfile []string + histfileSnapshot map[string][]string + history map[string][]string +} + +func (s *strategyRecentBash) init() { + s.histfileSnapshot = map[string][]string{} + s.history = map[string][]string{} +} + +func (s *strategyRecentBash) GetTitleAndDescription() (string, string) { + return "recent (bash-like)", "Behave like bash" +} + +func (s *strategyRecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string { + // populate the local history from histfile + if s.histfileSnapshot[strippedRecord.SessionID] == nil { + s.histfileSnapshot[strippedRecord.SessionID] = s.histfile + } + return append(s.history[strippedRecord.SessionID], s.histfileSnapshot[strippedRecord.SessionID]...) +} + +func (s *strategyRecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { + // remove previous occurance of record + for i, cmd := range s.history[record.SessionID] { + if cmd == record.CmdLine { + s.history[record.SessionID] = append(s.history[record.SessionID][:i], s.history[record.SessionID][i+1:]...) + } + } + // append new record + s.history[record.SessionID] = append([]string{record.CmdLine}, s.history[record.SessionID]...) + + if record.LastRecordOfSession { + // append history of the session to histfile and clear session history + s.histfile = append(s.history[record.SessionID], s.histfile...) + s.histfileSnapshot[record.SessionID] = nil + s.history[record.SessionID] = nil + } + return nil +} + +func (s *strategyRecentBash) ResetHistory() error { + s.init() + return nil +} diff --git a/cmd/evaluate/strategy-record-distance.go b/cmd/evaluate/strategy-record-distance.go index 31ed8e0..d2b8696 100644 --- a/cmd/evaluate/strategy-record-distance.go +++ b/cmd/evaluate/strategy-record-distance.go @@ -27,20 +27,16 @@ func (s *strategyRecordDistance) GetTitleAndDescription() (string, string) { return "record distance (depth:" + strconv.Itoa(s.maxDepth) + ";" + s.label + ")", "Use record distance to recommend commands" } -func (s *strategyRecordDistance) GetCandidates() []string { +func (s *strategyRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { if len(s.history) == 0 { return nil } - var prevRecord records.EnrichedRecord - prevRecord = s.history[0] - prevRecord.SetCmdLine("") - prevRecord.SetBeforeToAfter() var mapItems []strDistEntry for i, record := range s.history { if s.maxDepth != 0 && i > s.maxDepth { break } - distance := record.DistanceTo(prevRecord, s.distParams) + distance := record.DistanceTo(strippedRecord, s.distParams) mapItems = append(mapItems, strDistEntry{record.CmdLine, distance}) } sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go new file mode 100644 index 0000000..8373306 --- /dev/null +++ b/pkg/cfg/cfg.go @@ -0,0 +1,6 @@ +package cfg + +// Config struct +type Config struct { + Port int +} diff --git a/pkg/records/records.go b/pkg/records/records.go index c63950d..32ea789 100644 --- a/pkg/records/records.go +++ b/pkg/records/records.go @@ -89,12 +89,13 @@ type EnrichedRecord struct { Record // enriching fields - added "later" - Command string `json:"command"` - FirstWord string `json:"firstWord"` - Invalid bool `json:"invalid"` - SeqSessionID uint64 `json:"seqSessionId"` - DebugThisRecord bool `json:"debugThisRecord"` - Errors []string `json:"errors"` + Command string `json:"command"` + FirstWord string `json:"firstWord"` + Invalid bool `json:"invalid"` + SeqSessionID uint64 `json:"seqSessionId"` + LastRecordOfSession bool `json:"lastRecordOfSession"` + DebugThisRecord bool `json:"debugThisRecord"` + Errors []string `json:"errors"` // SeqSessionID uint64 `json:"seqSessionId,omitempty"` } @@ -213,6 +214,22 @@ func (r *EnrichedRecord) SetCmdLine(cmdLine string) { } } +// Stripped returns record stripped of all info that is not available during prediction +func Stripped(r EnrichedRecord) EnrichedRecord { + // clear the cmd itself + r.SetCmdLine("") + // replace after info with before info + r.PwdAfter = r.Pwd + r.RealPwdAfter = r.RealPwd + r.TimezoneAfter = r.TimezoneBefore + r.RealtimeAfter = r.RealtimeBefore + r.RealtimeAfterLocal = r.RealtimeBeforeLocal + // clear some more stuff + r.RealtimeDuration = 0 + r.LastRecordOfSession = false + return r +} + // SetBeforeToAfter - set "before" members to "after" members func (r *EnrichedRecord) SetBeforeToAfter() { r.Pwd = r.PwdAfter @@ -375,8 +392,3 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { return dist } - -// Config struct -type Config struct { - Port int -} From 79d7f1c45ef8f883ad75e8bd23472d720ca0774f Mon Sep 17 00:00:00 2001 From: Simon Let Date: Thu, 26 Sep 2019 01:35:23 +0200 Subject: [PATCH 09/13] restructure - move strategies to pkg, split evaluate and move parts to pkg --- cmd/evaluate/main.go | 425 ++---------------- cmd/evaluate/strategy-dummy.go | 24 - pkg/histanal/histeval.go | 246 ++++++++++ pkg/histanal/histload.go | 179 ++++++++ .../strat/directory-sensitive.go | 15 +- pkg/strat/dummy.go | 24 + .../strat/dynamic-record-distance.go | 38 +- .../strat/frequent.go | 14 +- .../strat/markov-chain-cmd.go | 26 +- .../strat/markov-chain.go | 26 +- .../strategy-random.go => pkg/strat/random.go | 20 +- .../strat/recent-bash.go | 16 +- .../strategy-recent.go => pkg/strat/recent.go | 12 +- .../strat/record-distance.go | 28 +- pkg/strat/strat.go | 44 ++ 15 files changed, 624 insertions(+), 513 deletions(-) delete mode 100644 cmd/evaluate/strategy-dummy.go create mode 100644 pkg/histanal/histeval.go create mode 100644 pkg/histanal/histload.go rename cmd/evaluate/strategy-directory-sensitive.go => pkg/strat/directory-sensitive.go (63%) create mode 100644 pkg/strat/dummy.go rename cmd/evaluate/strategy-dynamic-record-distance.go => pkg/strat/dynamic-record-distance.go (58%) rename cmd/evaluate/strategy-frequent.go => pkg/strat/frequent.go (64%) rename cmd/evaluate/strategy-markov-chain-cmd.go => pkg/strat/markov-chain-cmd.go (75%) rename cmd/evaluate/strategy-markov-chain.go => pkg/strat/markov-chain.go (68%) rename cmd/evaluate/strategy-random.go => pkg/strat/random.go (62%) rename cmd/evaluate/strategy-recent-bash.go => pkg/strat/recent-bash.go (75%) rename cmd/evaluate/strategy-recent.go => pkg/strat/recent.go (59%) rename cmd/evaluate/strategy-record-distance.go => pkg/strat/record-distance.go (53%) create mode 100644 pkg/strat/strat.go diff --git a/cmd/evaluate/main.go b/cmd/evaluate/main.go index 6e013cc..7ae217f 100644 --- a/cmd/evaluate/main.go +++ b/cmd/evaluate/main.go @@ -1,23 +1,16 @@ package main import ( - "bufio" - "bytes" - "encoding/json" "flag" "fmt" - "io/ioutil" "log" - "math/rand" "os" - "os/exec" "os/user" "path/filepath" + "github.com/curusarn/resh/pkg/histanal" "github.com/curusarn/resh/pkg/records" - "github.com/jpillora/longestcommon" - - "github.com/schollz/progressbar" + "github.com/curusarn/resh/pkg/strat" ) // Version from git set during build @@ -81,27 +74,20 @@ func main() { } } - evaluator := evaluator{sanitizedInput: *sanitizedInput, maxCandidates: maxCandidates, - BatchMode: batchMode, skipFailedCmds: *skipFailedCmds, debugRecords: *debugRecords} + var evaluator histanal.HistEval if batchMode { - err := evaluator.initBatchMode(*input, *inputDataRoot) - if err != nil { - log.Fatal("Evaluator initBatchMode() error:", err) - } + evaluator = histanal.NewHistEvalBatchMode(*input, *inputDataRoot, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) } else { - err := evaluator.init(*input) - if err != nil { - log.Fatal("Evaluator init() error:", err) - } + evaluator = histanal.NewHistEval(*input, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) } - var simpleStrategies []ISimpleStrategy - var strategies []IStrategy + var simpleStrategies []strat.ISimpleStrategy + var strategies []strat.IStrategy // dummy := strategyDummy{} // simpleStrategies = append(simpleStrategies, &dummy) - simpleStrategies = append(simpleStrategies, &strategyRecent{}) + simpleStrategies = append(simpleStrategies, &strat.Recent{}) // frequent := strategyFrequent{} // frequent.init() @@ -111,401 +97,56 @@ func main() { // random.init() // simpleStrategies = append(simpleStrategies, &random) - directory := strategyDirectorySensitive{} - directory.init() + directory := strat.DirectorySensitive{} + directory.Init() simpleStrategies = append(simpleStrategies, &directory) - dynamicDistG := strategyDynamicRecordDistance{ - maxDepth: 3000, - distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1, Git: 10}, - label: "10*pwd,10*realpwd,session,time,10*git", + dynamicDistG := strat.DynamicRecordDistance{ + MaxDepth: 3000, + DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1, Git: 10}, + Label: "10*pwd,10*realpwd,session,time,10*git", } - dynamicDistG.init() + dynamicDistG.Init() strategies = append(strategies, &dynamicDistG) - distanceStaticBest := strategyRecordDistance{ - maxDepth: 3000, - distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, - label: "10*pwd,10*realpwd,session,time", + distanceStaticBest := strat.RecordDistance{ + MaxDepth: 3000, + DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, + Label: "10*pwd,10*realpwd,session,time", } strategies = append(strategies, &distanceStaticBest) - recentBash := strategyRecentBash{} - recentBash.init() + recentBash := strat.RecentBash{} + recentBash.Init() strategies = append(strategies, &recentBash) if *slow { - markovCmd := strategyMarkovChainCmd{order: 1} - markovCmd.init() + markovCmd := strat.MarkovChainCmd{Order: 1} + markovCmd.Init() - markovCmd2 := strategyMarkovChainCmd{order: 2} - markovCmd2.init() + markovCmd2 := strat.MarkovChainCmd{Order: 2} + markovCmd2.Init() - markov := strategyMarkovChain{order: 1} - markov.init() + markov := strat.MarkovChain{Order: 1} + markov.Init() - markov2 := strategyMarkovChain{order: 2} - markov2.init() + markov2 := strat.MarkovChain{Order: 2} + markov2.Init() simpleStrategies = append(simpleStrategies, &markovCmd2, &markovCmd, &markov2, &markov) } - for _, strat := range simpleStrategies { - strategies = append(strategies, NewSimpleStrategyWrapper(strat)) + for _, strategy := range simpleStrategies { + strategies = append(strategies, strat.NewSimpleStrategyWrapper(strategy)) } for _, strat := range strategies { - err := evaluator.evaluate(strat) + err := evaluator.Evaluate(strat) if err != nil { log.Println("Evaluator evaluate() error:", err) } } - evaluator.calculateStatsAndPlot(*plottingScript) -} - -type ISimpleStrategy interface { - GetTitleAndDescription() (string, string) - GetCandidates() []string - AddHistoryRecord(record *records.EnrichedRecord) error - ResetHistory() error -} - -type IStrategy interface { - GetTitleAndDescription() (string, string) - GetCandidates(r records.EnrichedRecord) []string - AddHistoryRecord(record *records.EnrichedRecord) error - ResetHistory() error -} - -type simpleStrategyWrapper struct { - strategy ISimpleStrategy -} - -// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy -func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper { - return &simpleStrategyWrapper{strategy: strategy} -} - -func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) { - return s.strategy.GetTitleAndDescription() -} - -func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string { - return s.strategy.GetCandidates() -} - -func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error { - return s.strategy.AddHistoryRecord(r) -} - -func (s *simpleStrategyWrapper) ResetHistory() error { - return s.strategy.ResetHistory() -} - -type matchJSON struct { - Match bool - Distance int - CharsRecalled int -} - -type multiMatchItemJSON struct { - Distance int - CharsRecalled int -} - -type multiMatchJSON struct { - Match bool - Entries []multiMatchItemJSON -} - -type strategyJSON struct { - Title string - Description string - Matches []matchJSON - PrefixMatches []multiMatchJSON -} - -type deviceRecords struct { - Name string - Records []records.EnrichedRecord -} - -type userRecords struct { - Name string - Devices []deviceRecords -} - -type evaluator struct { - sanitizedInput bool - BatchMode bool - maxCandidates int - skipFailedCmds bool - debugRecords float64 - UsersRecords []userRecords - Strategies []strategyJSON -} - -func (e *evaluator) initBatchMode(input string, inputDataRoot string) error { - e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) - e.preprocessRecords() - return nil -} - -func (e *evaluator) init(inputPath string) error { - records := e.loadHistoryRecords(inputPath) - device := deviceRecords{Records: records} - user := userRecords{} - user.Devices = append(user.Devices, device) - e.UsersRecords = append(e.UsersRecords, user) - e.preprocessRecords() - return nil -} - -func (e *evaluator) calculateStatsAndPlot(scriptName string) { - evalJSON, err := json.Marshal(e) - if err != nil { - log.Fatal("json marshal error", err) - } - buffer := bytes.Buffer{} - buffer.Write(evalJSON) - // run python script to stat and plot/ - cmd := exec.Command(scriptName) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = &buffer - err = cmd.Run() - if err != nil { - log.Printf("Command finished with error: %v", err) - } -} - -func (e *evaluator) preprocessDeviceRecords(device deviceRecords) deviceRecords { - sessionIDs := map[string]uint64{} - var nextID uint64 - nextID = 1 // start with 1 because 0 won't get saved to json - for k, record := range device.Records { - id, found := sessionIDs[record.SessionID] - if found == false { - id = nextID - sessionIDs[record.SessionID] = id - nextID++ - } - device.Records[k].SeqSessionID = id - // assert - if record.Sanitized != e.sanitizedInput { - if e.sanitizedInput { - log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") - } - log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") - } - device.Records[k].SeqSessionID = id - if e.debugRecords > 0 && rand.Float64() < e.debugRecords { - device.Records[k].DebugThisRecord = true - } - } - // sort.SliceStable(device.Records, func(x, y int) bool { - // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { - // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal - // } - // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID - // }) - - // iterate from back and mark last record of each session - sessionIDSet := map[string]bool{} - for i := len(device.Records) - 1; i >= 0; i-- { - var record *records.EnrichedRecord - record = &device.Records[i] - if sessionIDSet[record.SessionID] { - continue - } - sessionIDSet[record.SessionID] = true - record.LastRecordOfSession = true - } - return device -} - -// enrich records and add sequential session ID -func (e *evaluator) preprocessRecords() { - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) - } - } -} - -func (e *evaluator) evaluate(strategy IStrategy) error { - title, description := strategy.GetTitleAndDescription() - log.Println("Evaluating strategy:", title, "-", description) - strategyData := strategyJSON{Title: title, Description: description} - for i := range e.UsersRecords { - for j := range e.UsersRecords[i].Devices { - bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) - var prevRecord records.EnrichedRecord - for _, record := range e.UsersRecords[i].Devices[j].Records { - if e.skipFailedCmds && record.ExitCode != 0 { - continue - } - candidates := strategy.GetCandidates(records.Stripped(record)) - if record.DebugThisRecord { - log.Println() - log.Println("===================================================") - log.Println("STRATEGY:", title, "-", description) - log.Println("===================================================") - log.Println("Previous record:") - if prevRecord.RealtimeBefore == 0 { - log.Println("== NIL") - } else { - rec, _ := prevRecord.ToString() - log.Println(rec) - } - log.Println("---------------------------------------------------") - log.Println("Recommendations for:") - rec, _ := record.ToString() - log.Println(rec) - log.Println("---------------------------------------------------") - for i, candidate := range candidates { - if i > 10 { - break - } - log.Println(string(candidate)) - } - log.Println("===================================================") - } - - matchFound := false - longestPrefixMatchLength := 0 - multiMatch := multiMatchJSON{} - for i, candidate := range candidates { - // make an option (--calculate-total) to turn this on/off ? - // if i >= e.maxCandidates { - // break - // } - commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine})) - if commonPrefixLength > longestPrefixMatchLength { - longestPrefixMatchLength = commonPrefixLength - prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength} - multiMatch.Match = true - multiMatch.Entries = append(multiMatch.Entries, prefixMatch) - } - if candidate == record.CmdLine { - match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} - matchFound = true - strategyData.Matches = append(strategyData.Matches, match) - strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) - break - } - } - if matchFound == false { - strategyData.Matches = append(strategyData.Matches, matchJSON{}) - strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) - } - err := strategy.AddHistoryRecord(&record) - if err != nil { - log.Println("Error while evauating", err) - return err - } - bar.Add(1) - prevRecord = record - } - strategy.ResetHistory() - fmt.Println() - } - } - e.Strategies = append(e.Strategies, strategyData) - return nil -} - -func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords { - var records []userRecords - info, err := os.Stat(dataRootPath) - if err != nil { - log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")") - } - if info.IsDir() == false { - log.Fatal("Error:", dataRootPath, "is not a directory - exiting!") - } - users, err := ioutil.ReadDir(dataRootPath) - if err != nil { - log.Fatal("Could not read directory:", dataRootPath) - } - fmt.Println("Listing users in <", dataRootPath, ">...") - for _, user := range users { - userRecords := userRecords{Name: user.Name()} - userFullPath := filepath.Join(dataRootPath, user.Name()) - if user.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.") - continue - } - fmt.Println() - fmt.Printf("*- %s\n", user.Name()) - devices, err := ioutil.ReadDir(userFullPath) - if err != nil { - log.Fatal("Could not read directory:", userFullPath) - } - for _, device := range devices { - deviceRecords := deviceRecords{Name: device.Name()} - deviceFullPath := filepath.Join(userFullPath, device.Name()) - if device.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.") - continue - } - fmt.Printf(" \\- %s\n", device.Name()) - files, err := ioutil.ReadDir(deviceFullPath) - if err != nil { - log.Fatal("Could not read directory:", deviceFullPath) - } - for _, file := range files { - fileFullPath := filepath.Join(deviceFullPath, file.Name()) - if file.Name() == fname { - fmt.Printf(" \\- %s - loading ...", file.Name()) - // load the data - deviceRecords.Records = e.loadHistoryRecords(fileFullPath) - fmt.Println(" OK ✓") - } else { - fmt.Printf(" \\- %s - skipped\n", file.Name()) - } - } - userRecords.Devices = append(userRecords.Devices, deviceRecords) - } - records = append(records, userRecords) - } - return records -} - -func (e *evaluator) loadHistoryRecords(fname string) []records.EnrichedRecord { - file, err := os.Open(fname) - if err != nil { - log.Fatal("Open() resh history file error:", err) - } - defer file.Close() - - var recs []records.EnrichedRecord - scanner := bufio.NewScanner(file) - for scanner.Scan() { - record := records.Record{} - fallbackRecord := records.FallbackRecord{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) - } - record = records.ConvertRecord(&fallbackRecord) - } - if e.sanitizedInput == false { - if record.CmdLength != 0 { - log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?") - } - record.CmdLength = len(record.CmdLine) - } - if record.CmdLength == 0 { - log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") - } - recs = append(recs, record.Enrich()) - } - return recs + evaluator.CalculateStatsAndPlot(*plottingScript) } diff --git a/cmd/evaluate/strategy-dummy.go b/cmd/evaluate/strategy-dummy.go deleted file mode 100644 index 9d20779..0000000 --- a/cmd/evaluate/strategy-dummy.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import "github.com/curusarn/resh/pkg/records" - -type strategyDummy struct { - history []string -} - -func (s *strategyDummy) GetTitleAndDescription() (string, string) { - return "dummy", "Return empty candidate list" -} - -func (s *strategyDummy) GetCandidates() []string { - return nil -} - -func (s *strategyDummy) AddHistoryRecord(record *records.EnrichedRecord) error { - s.history = append(s.history, record.CmdLine) - return nil -} - -func (s *strategyDummy) ResetHistory() error { - return nil -} diff --git a/pkg/histanal/histeval.go b/pkg/histanal/histeval.go new file mode 100644 index 0000000..4d19779 --- /dev/null +++ b/pkg/histanal/histeval.go @@ -0,0 +1,246 @@ +package histanal + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "math/rand" + "os" + "os/exec" + + "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/pkg/strat" + "github.com/jpillora/longestcommon" + + "github.com/schollz/progressbar" +) + +type matchJSON struct { + Match bool + Distance int + CharsRecalled int +} + +type multiMatchItemJSON struct { + Distance int + CharsRecalled int +} + +type multiMatchJSON struct { + Match bool + Entries []multiMatchItemJSON +} + +type strategyJSON struct { + Title string + Description string + Matches []matchJSON + PrefixMatches []multiMatchJSON +} + +// HistEval evaluates history +type HistEval struct { + HistLoad + BatchMode bool + maxCandidates int + Strategies []strategyJSON +} + +// NewHistEval constructs new HistEval +func NewHistEval(inputPath string, + maxCandidates int, skipFailedCmds bool, + debugRecords float64, sanitizedInput bool) HistEval { + + e := HistEval{ + HistLoad: HistLoad{ + skipFailedCmds: skipFailedCmds, + debugRecords: debugRecords, + sanitizedInput: sanitizedInput, + }, + maxCandidates: maxCandidates, + BatchMode: false, + } + records := e.loadHistoryRecords(inputPath) + device := deviceRecords{Records: records} + user := userRecords{} + user.Devices = append(user.Devices, device) + e.UsersRecords = append(e.UsersRecords, user) + e.preprocessRecords() + return e +} + +// NewHistEvalBatchMode constructs new HistEval in batch mode +func NewHistEvalBatchMode(input string, inputDataRoot string, + maxCandidates int, skipFailedCmds bool, + debugRecords float64, sanitizedInput bool) HistEval { + + e := HistEval{ + HistLoad: HistLoad{ + skipFailedCmds: skipFailedCmds, + debugRecords: debugRecords, + sanitizedInput: sanitizedInput, + }, + maxCandidates: maxCandidates, + BatchMode: false, + } + e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) + e.preprocessRecords() + return e +} + +func (e *HistEval) preprocessDeviceRecords(device deviceRecords) deviceRecords { + sessionIDs := map[string]uint64{} + var nextID uint64 + nextID = 1 // start with 1 because 0 won't get saved to json + for k, record := range device.Records { + id, found := sessionIDs[record.SessionID] + if found == false { + id = nextID + sessionIDs[record.SessionID] = id + nextID++ + } + device.Records[k].SeqSessionID = id + // assert + if record.Sanitized != e.sanitizedInput { + if e.sanitizedInput { + log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") + } + log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") + } + device.Records[k].SeqSessionID = id + if e.debugRecords > 0 && rand.Float64() < e.debugRecords { + device.Records[k].DebugThisRecord = true + } + } + // sort.SliceStable(device.Records, func(x, y int) bool { + // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { + // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal + // } + // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID + // }) + + // iterate from back and mark last record of each session + sessionIDSet := map[string]bool{} + for i := len(device.Records) - 1; i >= 0; i-- { + var record *records.EnrichedRecord + record = &device.Records[i] + if sessionIDSet[record.SessionID] { + continue + } + sessionIDSet[record.SessionID] = true + record.LastRecordOfSession = true + } + return device +} + +// enrich records and add sequential session ID +func (e *HistEval) preprocessRecords() { + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) + } + } +} + +// Evaluate a given strategy +func (e *HistEval) Evaluate(strategy strat.IStrategy) error { + title, description := strategy.GetTitleAndDescription() + log.Println("Evaluating strategy:", title, "-", description) + strategyData := strategyJSON{Title: title, Description: description} + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) + var prevRecord records.EnrichedRecord + for _, record := range e.UsersRecords[i].Devices[j].Records { + if e.skipFailedCmds && record.ExitCode != 0 { + continue + } + candidates := strategy.GetCandidates(records.Stripped(record)) + if record.DebugThisRecord { + log.Println() + log.Println("===================================================") + log.Println("STRATEGY:", title, "-", description) + log.Println("===================================================") + log.Println("Previous record:") + if prevRecord.RealtimeBefore == 0 { + log.Println("== NIL") + } else { + rec, _ := prevRecord.ToString() + log.Println(rec) + } + log.Println("---------------------------------------------------") + log.Println("Recommendations for:") + rec, _ := record.ToString() + log.Println(rec) + log.Println("---------------------------------------------------") + for i, candidate := range candidates { + if i > 10 { + break + } + log.Println(string(candidate)) + } + log.Println("===================================================") + } + + matchFound := false + longestPrefixMatchLength := 0 + multiMatch := multiMatchJSON{} + for i, candidate := range candidates { + // make an option (--calculate-total) to turn this on/off ? + // if i >= e.maxCandidates { + // break + // } + commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine})) + if commonPrefixLength > longestPrefixMatchLength { + longestPrefixMatchLength = commonPrefixLength + prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength} + multiMatch.Match = true + multiMatch.Entries = append(multiMatch.Entries, prefixMatch) + } + if candidate == record.CmdLine { + match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} + matchFound = true + strategyData.Matches = append(strategyData.Matches, match) + strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) + break + } + } + if matchFound == false { + strategyData.Matches = append(strategyData.Matches, matchJSON{}) + strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) + } + err := strategy.AddHistoryRecord(&record) + if err != nil { + log.Println("Error while evauating", err) + return err + } + bar.Add(1) + prevRecord = record + } + strategy.ResetHistory() + fmt.Println() + } + } + e.Strategies = append(e.Strategies, strategyData) + return nil +} + +// CalculateStatsAndPlot results +func (e *HistEval) CalculateStatsAndPlot(scriptName string) { + evalJSON, err := json.Marshal(e) + if err != nil { + log.Fatal("json marshal error", err) + } + buffer := bytes.Buffer{} + buffer.Write(evalJSON) + // run python script to stat and plot/ + cmd := exec.Command(scriptName) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = &buffer + err = cmd.Run() + if err != nil { + log.Printf("Command finished with error: %v", err) + } +} diff --git a/pkg/histanal/histload.go b/pkg/histanal/histload.go new file mode 100644 index 0000000..c8f253b --- /dev/null +++ b/pkg/histanal/histload.go @@ -0,0 +1,179 @@ +package histanal + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "math/rand" + "os" + "path/filepath" + + "github.com/curusarn/resh/pkg/records" +) + +type deviceRecords struct { + Name string + Records []records.EnrichedRecord +} + +type userRecords struct { + Name string + Devices []deviceRecords +} + +// HistLoad loads history +type HistLoad struct { + UsersRecords []userRecords + skipFailedCmds bool + sanitizedInput bool + debugRecords float64 +} + +func (e *HistLoad) preprocessDeviceRecords(device deviceRecords) deviceRecords { + sessionIDs := map[string]uint64{} + var nextID uint64 + nextID = 1 // start with 1 because 0 won't get saved to json + for k, record := range device.Records { + id, found := sessionIDs[record.SessionID] + if found == false { + id = nextID + sessionIDs[record.SessionID] = id + nextID++ + } + device.Records[k].SeqSessionID = id + // assert + if record.Sanitized != e.sanitizedInput { + if e.sanitizedInput { + log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") + } + log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") + } + device.Records[k].SeqSessionID = id + if e.debugRecords > 0 && rand.Float64() < e.debugRecords { + device.Records[k].DebugThisRecord = true + } + } + // sort.SliceStable(device.Records, func(x, y int) bool { + // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { + // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal + // } + // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID + // }) + + // iterate from back and mark last record of each session + sessionIDSet := map[string]bool{} + for i := len(device.Records) - 1; i >= 0; i-- { + var record *records.EnrichedRecord + record = &device.Records[i] + if sessionIDSet[record.SessionID] { + continue + } + sessionIDSet[record.SessionID] = true + record.LastRecordOfSession = true + } + return device +} + +// enrich records and add sequential session ID +func (e *HistLoad) preprocessRecords() { + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) + } + } +} + +func (e *HistLoad) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords { + var records []userRecords + info, err := os.Stat(dataRootPath) + if err != nil { + log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")") + } + if info.IsDir() == false { + log.Fatal("Error:", dataRootPath, "is not a directory - exiting!") + } + users, err := ioutil.ReadDir(dataRootPath) + if err != nil { + log.Fatal("Could not read directory:", dataRootPath) + } + fmt.Println("Listing users in <", dataRootPath, ">...") + for _, user := range users { + userRecords := userRecords{Name: user.Name()} + userFullPath := filepath.Join(dataRootPath, user.Name()) + if user.IsDir() == false { + log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.") + continue + } + fmt.Println() + fmt.Printf("*- %s\n", user.Name()) + devices, err := ioutil.ReadDir(userFullPath) + if err != nil { + log.Fatal("Could not read directory:", userFullPath) + } + for _, device := range devices { + deviceRecords := deviceRecords{Name: device.Name()} + deviceFullPath := filepath.Join(userFullPath, device.Name()) + if device.IsDir() == false { + log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.") + continue + } + fmt.Printf(" \\- %s\n", device.Name()) + files, err := ioutil.ReadDir(deviceFullPath) + if err != nil { + log.Fatal("Could not read directory:", deviceFullPath) + } + for _, file := range files { + fileFullPath := filepath.Join(deviceFullPath, file.Name()) + if file.Name() == fname { + fmt.Printf(" \\- %s - loading ...", file.Name()) + // load the data + deviceRecords.Records = e.loadHistoryRecords(fileFullPath) + fmt.Println(" OK ✓") + } else { + fmt.Printf(" \\- %s - skipped\n", file.Name()) + } + } + userRecords.Devices = append(userRecords.Devices, deviceRecords) + } + records = append(records, userRecords) + } + return records +} + +func (e *HistLoad) loadHistoryRecords(fname string) []records.EnrichedRecord { + file, err := os.Open(fname) + if err != nil { + log.Fatal("Open() resh history file error:", err) + } + defer file.Close() + + var recs []records.EnrichedRecord + scanner := bufio.NewScanner(file) + for scanner.Scan() { + record := records.Record{} + fallbackRecord := records.FallbackRecord{} + line := scanner.Text() + err = json.Unmarshal([]byte(line), &record) + if err != nil { + err = json.Unmarshal([]byte(line), &fallbackRecord) + if err != nil { + log.Println("Line:", line) + log.Fatal("Decoding error:", err) + } + record = records.ConvertRecord(&fallbackRecord) + } + if e.sanitizedInput == false { + if record.CmdLength != 0 { + log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?") + } + record.CmdLength = len(record.CmdLine) + } + if record.CmdLength == 0 { + log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") + } + recs = append(recs, record.Enrich()) + } + return recs +} diff --git a/cmd/evaluate/strategy-directory-sensitive.go b/pkg/strat/directory-sensitive.go similarity index 63% rename from cmd/evaluate/strategy-directory-sensitive.go rename to pkg/strat/directory-sensitive.go index a05deac..87edaec 100644 --- a/cmd/evaluate/strategy-directory-sensitive.go +++ b/pkg/strat/directory-sensitive.go @@ -1,25 +1,25 @@ -package main +package strat import "github.com/curusarn/resh/pkg/records" -type strategyDirectorySensitive struct { +type DirectorySensitive struct { history map[string][]string lastPwd string } -func (s *strategyDirectorySensitive) init() { +func (s *DirectorySensitive) Init() { s.history = map[string][]string{} } -func (s *strategyDirectorySensitive) GetTitleAndDescription() (string, string) { +func (s *DirectorySensitive) GetTitleAndDescription() (string, string) { return "directory sensitive (recent)", "Use recent commands executed is the same directory" } -func (s *strategyDirectorySensitive) GetCandidates() []string { +func (s *DirectorySensitive) GetCandidates() []string { return s.history[s.lastPwd] } -func (s *strategyDirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *DirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error { // work on history for PWD pwd := record.Pwd // remove previous occurance of record @@ -34,7 +34,8 @@ func (s *strategyDirectorySensitive) AddHistoryRecord(record *records.EnrichedRe return nil } -func (s *strategyDirectorySensitive) ResetHistory() error { +func (s *DirectorySensitive) ResetHistory() error { + s.Init() s.history = map[string][]string{} return nil } diff --git a/pkg/strat/dummy.go b/pkg/strat/dummy.go new file mode 100644 index 0000000..b21a60d --- /dev/null +++ b/pkg/strat/dummy.go @@ -0,0 +1,24 @@ +package strat + +import "github.com/curusarn/resh/pkg/records" + +type Dummy struct { + history []string +} + +func (s *Dummy) GetTitleAndDescription() (string, string) { + return "dummy", "Return empty candidate list" +} + +func (s *Dummy) GetCandidates() []string { + return nil +} + +func (s *Dummy) AddHistoryRecord(record *records.EnrichedRecord) error { + s.history = append(s.history, record.CmdLine) + return nil +} + +func (s *Dummy) ResetHistory() error { + return nil +} diff --git a/cmd/evaluate/strategy-dynamic-record-distance.go b/pkg/strat/dynamic-record-distance.go similarity index 58% rename from cmd/evaluate/strategy-dynamic-record-distance.go rename to pkg/strat/dynamic-record-distance.go index 0a107e9..4a3be10 100644 --- a/cmd/evaluate/strategy-dynamic-record-distance.go +++ b/pkg/strat/dynamic-record-distance.go @@ -1,4 +1,4 @@ -package main +package strat import ( "math" @@ -8,14 +8,14 @@ import ( "github.com/curusarn/resh/pkg/records" ) -type strategyDynamicRecordDistance struct { +type DynamicRecordDistance struct { history []records.EnrichedRecord - distParams records.DistParams + DistParams records.DistParams pwdHistogram map[string]int realPwdHistogram map[string]int gitOriginHistogram map[string]int - maxDepth int - label string + MaxDepth int + Label string } type strDynDistEntry struct { @@ -23,36 +23,36 @@ type strDynDistEntry struct { distance float64 } -func (s *strategyDynamicRecordDistance) init() { +func (s *DynamicRecordDistance) Init() { s.history = nil s.pwdHistogram = map[string]int{} s.realPwdHistogram = map[string]int{} s.gitOriginHistogram = map[string]int{} } -func (s *strategyDynamicRecordDistance) GetTitleAndDescription() (string, string) { - return "dynamic record distance (depth:" + strconv.Itoa(s.maxDepth) + ";" + s.label + ")", "Use TF-IDF record distance to recommend commands" +func (s *DynamicRecordDistance) GetTitleAndDescription() (string, string) { + return "dynamic record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use TF-IDF record distance to recommend commands" } -func (s *strategyDynamicRecordDistance) idf(count int) float64 { +func (s *DynamicRecordDistance) idf(count int) float64 { return math.Log(float64(len(s.history)) / float64(count)) } -func (s *strategyDynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { +func (s *DynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { if len(s.history) == 0 { return nil } var mapItems []strDynDistEntry for i, record := range s.history { - if s.maxDepth != 0 && i > s.maxDepth { + if s.MaxDepth != 0 && i > s.MaxDepth { break } distParams := records.DistParams{ - Pwd: s.distParams.Pwd * s.idf(s.pwdHistogram[strippedRecord.PwdAfter]), - RealPwd: s.distParams.RealPwd * s.idf(s.realPwdHistogram[strippedRecord.RealPwdAfter]), - Git: s.distParams.Git * s.idf(s.gitOriginHistogram[strippedRecord.GitOriginRemote]), - Time: s.distParams.Time, - SessionID: s.distParams.SessionID, + Pwd: s.DistParams.Pwd * s.idf(s.pwdHistogram[strippedRecord.PwdAfter]), + RealPwd: s.DistParams.RealPwd * s.idf(s.realPwdHistogram[strippedRecord.RealPwdAfter]), + Git: s.DistParams.Git * s.idf(s.gitOriginHistogram[strippedRecord.GitOriginRemote]), + Time: s.DistParams.Time, + SessionID: s.DistParams.SessionID, } distance := record.DistanceTo(strippedRecord, distParams) mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance}) @@ -70,7 +70,7 @@ func (s *strategyDynamicRecordDistance) GetCandidates(strippedRecord records.Enr return hist } -func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *DynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { // append record to front s.history = append([]records.EnrichedRecord{*record}, s.history...) s.pwdHistogram[record.Pwd]++ @@ -79,7 +79,7 @@ func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *records.Enriche return nil } -func (s *strategyDynamicRecordDistance) ResetHistory() error { - s.init() +func (s *DynamicRecordDistance) ResetHistory() error { + s.Init() return nil } diff --git a/cmd/evaluate/strategy-frequent.go b/pkg/strat/frequent.go similarity index 64% rename from cmd/evaluate/strategy-frequent.go rename to pkg/strat/frequent.go index f081742..6525b9f 100644 --- a/cmd/evaluate/strategy-frequent.go +++ b/pkg/strat/frequent.go @@ -1,4 +1,4 @@ -package main +package strat import ( "sort" @@ -6,7 +6,7 @@ import ( "github.com/curusarn/resh/pkg/records" ) -type strategyFrequent struct { +type Frequent struct { history map[string]int } @@ -15,15 +15,15 @@ type strFrqEntry struct { count int } -func (s *strategyFrequent) init() { +func (s *Frequent) init() { s.history = map[string]int{} } -func (s *strategyFrequent) GetTitleAndDescription() (string, string) { +func (s *Frequent) GetTitleAndDescription() (string, string) { return "frequent", "Use frequent commands" } -func (s *strategyFrequent) GetCandidates() []string { +func (s *Frequent) GetCandidates() []string { var mapItems []strFrqEntry for cmdLine, count := range s.history { mapItems = append(mapItems, strFrqEntry{cmdLine, count}) @@ -36,12 +36,12 @@ func (s *strategyFrequent) GetCandidates() []string { return hist } -func (s *strategyFrequent) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *Frequent) AddHistoryRecord(record *records.EnrichedRecord) error { s.history[record.CmdLine]++ return nil } -func (s *strategyFrequent) ResetHistory() error { +func (s *Frequent) ResetHistory() error { s.init() return nil } diff --git a/cmd/evaluate/strategy-markov-chain-cmd.go b/pkg/strat/markov-chain-cmd.go similarity index 75% rename from cmd/evaluate/strategy-markov-chain-cmd.go rename to pkg/strat/markov-chain-cmd.go index 7c4ae7f..1bd1ded 100644 --- a/cmd/evaluate/strategy-markov-chain-cmd.go +++ b/pkg/strat/markov-chain-cmd.go @@ -1,4 +1,4 @@ -package main +package strat import ( "sort" @@ -8,8 +8,8 @@ import ( "github.com/mb-14/gomarkov" ) -type strategyMarkovChainCmd struct { - order int +type MarkovChainCmd struct { + Order int history []strMarkCmdHistoryEntry historyCmds []string } @@ -24,24 +24,24 @@ type strMarkCmdEntry struct { transProb float64 } -func (s *strategyMarkovChainCmd) init() { +func (s *MarkovChainCmd) Init() { s.history = nil s.historyCmds = nil } -func (s *strategyMarkovChainCmd) GetTitleAndDescription() (string, string) { - return "command-based markov chain (order " + strconv.Itoa(s.order) + ")", "Use command-based markov chain to recommend commands" +func (s *MarkovChainCmd) GetTitleAndDescription() (string, string) { + return "command-based markov chain (order " + strconv.Itoa(s.Order) + ")", "Use command-based markov chain to recommend commands" } -func (s *strategyMarkovChainCmd) GetCandidates() []string { - if len(s.history) < s.order { +func (s *MarkovChainCmd) GetCandidates() []string { + if len(s.history) < s.Order { var hist []string for _, item := range s.history { hist = append(hist, item.cmdLine) } return hist } - chain := gomarkov.NewChain(s.order) + chain := gomarkov.NewChain(s.Order) chain.Add(s.historyCmds) @@ -52,7 +52,7 @@ func (s *strategyMarkovChainCmd) GetCandidates() []string { continue } cmdsSet[cmd] = true - prob, _ := chain.TransitionProbability(cmd, s.historyCmds[len(s.historyCmds)-s.order:]) + prob, _ := chain.TransitionProbability(cmd, s.historyCmds[len(s.historyCmds)-s.Order:]) entries = append(entries, strMarkCmdEntry{cmd: cmd, transProb: prob}) } sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) @@ -78,14 +78,14 @@ func (s *strategyMarkovChainCmd) GetCandidates() []string { return hist } -func (s *strategyMarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *MarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command}) s.historyCmds = append(s.historyCmds, record.Command) // s.historySet[record.CmdLine] = true return nil } -func (s *strategyMarkovChainCmd) ResetHistory() error { - s.init() +func (s *MarkovChainCmd) ResetHistory() error { + s.Init() return nil } diff --git a/cmd/evaluate/strategy-markov-chain.go b/pkg/strat/markov-chain.go similarity index 68% rename from cmd/evaluate/strategy-markov-chain.go rename to pkg/strat/markov-chain.go index 25876b9..07006e0 100644 --- a/cmd/evaluate/strategy-markov-chain.go +++ b/pkg/strat/markov-chain.go @@ -1,4 +1,4 @@ -package main +package strat import ( "sort" @@ -8,8 +8,8 @@ import ( "github.com/mb-14/gomarkov" ) -type strategyMarkovChain struct { - order int +type MarkovChain struct { + Order int history []string } @@ -18,19 +18,19 @@ type strMarkEntry struct { transProb float64 } -func (s *strategyMarkovChain) init() { +func (s *MarkovChain) Init() { s.history = nil } -func (s *strategyMarkovChain) GetTitleAndDescription() (string, string) { - return "markov chain (order " + strconv.Itoa(s.order) + ")", "Use markov chain to recommend commands" +func (s *MarkovChain) GetTitleAndDescription() (string, string) { + return "markov chain (order " + strconv.Itoa(s.Order) + ")", "Use markov chain to recommend commands" } -func (s *strategyMarkovChain) GetCandidates() []string { - if len(s.history) < s.order { +func (s *MarkovChain) GetCandidates() []string { + if len(s.history) < s.Order { return s.history } - chain := gomarkov.NewChain(s.order) + chain := gomarkov.NewChain(s.Order) chain.Add(s.history) @@ -41,7 +41,7 @@ func (s *strategyMarkovChain) GetCandidates() []string { continue } cmdLinesSet[cmdLine] = true - prob, _ := chain.TransitionProbability(cmdLine, s.history[len(s.history)-s.order:]) + prob, _ := chain.TransitionProbability(cmdLine, s.history[len(s.history)-s.Order:]) entries = append(entries, strMarkEntry{cmdLine: cmdLine, transProb: prob}) } sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) @@ -58,13 +58,13 @@ func (s *strategyMarkovChain) GetCandidates() []string { return hist } -func (s *strategyMarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *MarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, record.CmdLine) // s.historySet[record.CmdLine] = true return nil } -func (s *strategyMarkovChain) ResetHistory() error { - s.init() +func (s *MarkovChain) ResetHistory() error { + s.Init() return nil } diff --git a/cmd/evaluate/strategy-random.go b/pkg/strat/random.go similarity index 62% rename from cmd/evaluate/strategy-random.go rename to pkg/strat/random.go index f8a59eb..27e959c 100644 --- a/cmd/evaluate/strategy-random.go +++ b/pkg/strat/random.go @@ -1,4 +1,4 @@ -package main +package strat import ( "math/rand" @@ -7,27 +7,27 @@ import ( "github.com/curusarn/resh/pkg/records" ) -type strategyRandom struct { - candidatesSize int +type Random struct { + CandidatesSize int history []string historySet map[string]bool } -func (s *strategyRandom) init() { +func (s *Random) Init() { s.history = nil s.historySet = map[string]bool{} } -func (s *strategyRandom) GetTitleAndDescription() (string, string) { +func (s *Random) GetTitleAndDescription() (string, string) { return "random", "Use random commands" } -func (s *strategyRandom) GetCandidates() []string { +func (s *Random) GetCandidates() []string { seed := time.Now().UnixNano() rand.Seed(seed) var candidates []string candidateSet := map[string]bool{} - for len(candidates) < s.candidatesSize && len(candidates)*2 < len(s.historySet) { + for len(candidates) < s.CandidatesSize && len(candidates)*2 < len(s.historySet) { x := rand.Intn(len(s.history)) candidate := s.history[x] if candidateSet[candidate] == false { @@ -39,13 +39,13 @@ func (s *strategyRandom) GetCandidates() []string { return candidates } -func (s *strategyRandom) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *Random) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append([]string{record.CmdLine}, s.history...) s.historySet[record.CmdLine] = true return nil } -func (s *strategyRandom) ResetHistory() error { - s.init() +func (s *Random) ResetHistory() error { + s.Init() return nil } diff --git a/cmd/evaluate/strategy-recent-bash.go b/pkg/strat/recent-bash.go similarity index 75% rename from cmd/evaluate/strategy-recent-bash.go rename to pkg/strat/recent-bash.go index 7a83632..c5319ce 100644 --- a/cmd/evaluate/strategy-recent-bash.go +++ b/pkg/strat/recent-bash.go @@ -1,23 +1,23 @@ -package main +package strat import "github.com/curusarn/resh/pkg/records" -type strategyRecentBash struct { +type RecentBash struct { histfile []string histfileSnapshot map[string][]string history map[string][]string } -func (s *strategyRecentBash) init() { +func (s *RecentBash) Init() { s.histfileSnapshot = map[string][]string{} s.history = map[string][]string{} } -func (s *strategyRecentBash) GetTitleAndDescription() (string, string) { +func (s *RecentBash) GetTitleAndDescription() (string, string) { return "recent (bash-like)", "Behave like bash" } -func (s *strategyRecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string { +func (s *RecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string { // populate the local history from histfile if s.histfileSnapshot[strippedRecord.SessionID] == nil { s.histfileSnapshot[strippedRecord.SessionID] = s.histfile @@ -25,7 +25,7 @@ func (s *strategyRecentBash) GetCandidates(strippedRecord records.EnrichedRecord return append(s.history[strippedRecord.SessionID], s.histfileSnapshot[strippedRecord.SessionID]...) } -func (s *strategyRecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *RecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { // remove previous occurance of record for i, cmd := range s.history[record.SessionID] { if cmd == record.CmdLine { @@ -44,7 +44,7 @@ func (s *strategyRecentBash) AddHistoryRecord(record *records.EnrichedRecord) er return nil } -func (s *strategyRecentBash) ResetHistory() error { - s.init() +func (s *RecentBash) ResetHistory() error { + s.Init() return nil } diff --git a/cmd/evaluate/strategy-recent.go b/pkg/strat/recent.go similarity index 59% rename from cmd/evaluate/strategy-recent.go rename to pkg/strat/recent.go index 967480c..68c8eff 100644 --- a/cmd/evaluate/strategy-recent.go +++ b/pkg/strat/recent.go @@ -1,20 +1,20 @@ -package main +package strat import "github.com/curusarn/resh/pkg/records" -type strategyRecent struct { +type Recent struct { history []string } -func (s *strategyRecent) GetTitleAndDescription() (string, string) { +func (s *Recent) GetTitleAndDescription() (string, string) { return "recent", "Use recent commands" } -func (s *strategyRecent) GetCandidates() []string { +func (s *Recent) GetCandidates() []string { return s.history } -func (s *strategyRecent) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *Recent) AddHistoryRecord(record *records.EnrichedRecord) error { // remove previous occurance of record for i, cmd := range s.history { if cmd == record.CmdLine { @@ -26,7 +26,7 @@ func (s *strategyRecent) AddHistoryRecord(record *records.EnrichedRecord) error return nil } -func (s *strategyRecent) ResetHistory() error { +func (s *Recent) ResetHistory() error { s.history = nil return nil } diff --git a/cmd/evaluate/strategy-record-distance.go b/pkg/strat/record-distance.go similarity index 53% rename from cmd/evaluate/strategy-record-distance.go rename to pkg/strat/record-distance.go index d2b8696..816922e 100644 --- a/cmd/evaluate/strategy-record-distance.go +++ b/pkg/strat/record-distance.go @@ -1,4 +1,4 @@ -package main +package strat import ( "sort" @@ -7,11 +7,11 @@ import ( "github.com/curusarn/resh/pkg/records" ) -type strategyRecordDistance struct { +type RecordDistance struct { history []records.EnrichedRecord - distParams records.DistParams - maxDepth int - label string + DistParams records.DistParams + MaxDepth int + Label string } type strDistEntry struct { @@ -19,24 +19,24 @@ type strDistEntry struct { distance float64 } -func (s *strategyRecordDistance) init() { +func (s *RecordDistance) Init() { s.history = nil } -func (s *strategyRecordDistance) GetTitleAndDescription() (string, string) { - return "record distance (depth:" + strconv.Itoa(s.maxDepth) + ";" + s.label + ")", "Use record distance to recommend commands" +func (s *RecordDistance) GetTitleAndDescription() (string, string) { + return "record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use record distance to recommend commands" } -func (s *strategyRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { +func (s *RecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { if len(s.history) == 0 { return nil } var mapItems []strDistEntry for i, record := range s.history { - if s.maxDepth != 0 && i > s.maxDepth { + if s.MaxDepth != 0 && i > s.MaxDepth { break } - distance := record.DistanceTo(strippedRecord, s.distParams) + distance := record.DistanceTo(strippedRecord, s.DistParams) mapItems = append(mapItems, strDistEntry{record.CmdLine, distance}) } sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) @@ -52,13 +52,13 @@ func (s *strategyRecordDistance) GetCandidates(strippedRecord records.EnrichedRe return hist } -func (s *strategyRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { +func (s *RecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { // append record to front s.history = append([]records.EnrichedRecord{*record}, s.history...) return nil } -func (s *strategyRecordDistance) ResetHistory() error { - s.init() +func (s *RecordDistance) ResetHistory() error { + s.Init() return nil } diff --git a/pkg/strat/strat.go b/pkg/strat/strat.go new file mode 100644 index 0000000..d5b1b2c --- /dev/null +++ b/pkg/strat/strat.go @@ -0,0 +1,44 @@ +package strat + +import ( + "github.com/curusarn/resh/pkg/records" +) + +type ISimpleStrategy interface { + GetTitleAndDescription() (string, string) + GetCandidates() []string + AddHistoryRecord(record *records.EnrichedRecord) error + ResetHistory() error +} + +type IStrategy interface { + GetTitleAndDescription() (string, string) + GetCandidates(r records.EnrichedRecord) []string + AddHistoryRecord(record *records.EnrichedRecord) error + ResetHistory() error +} + +type simpleStrategyWrapper struct { + strategy ISimpleStrategy +} + +// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy +func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper { + return &simpleStrategyWrapper{strategy: strategy} +} + +func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) { + return s.strategy.GetTitleAndDescription() +} + +func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string { + return s.strategy.GetCandidates() +} + +func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error { + return s.strategy.AddHistoryRecord(r) +} + +func (s *simpleStrategyWrapper) ResetHistory() error { + return s.strategy.ResetHistory() +} From 882e2f34a37c91ae5939971ff3decce2f0861a29 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Thu, 26 Sep 2019 02:00:45 +0200 Subject: [PATCH 10/13] Add a bunch of useless comments to make linter happy --- pkg/strat/directory-sensitive.go | 6 ++++++ pkg/strat/dummy.go | 5 +++++ pkg/strat/dynamic-record-distance.go | 6 ++++++ pkg/strat/frequent.go | 10 ++++++++-- pkg/strat/markov-chain-cmd.go | 6 ++++++ pkg/strat/markov-chain.go | 6 ++++++ pkg/strat/random.go | 6 ++++++ pkg/strat/recent-bash.go | 6 ++++++ pkg/strat/recent.go | 5 +++++ pkg/strat/record-distance.go | 6 ++++++ pkg/strat/strat.go | 2 ++ 11 files changed, 62 insertions(+), 2 deletions(-) diff --git a/pkg/strat/directory-sensitive.go b/pkg/strat/directory-sensitive.go index 87edaec..89d030e 100644 --- a/pkg/strat/directory-sensitive.go +++ b/pkg/strat/directory-sensitive.go @@ -2,23 +2,28 @@ package strat import "github.com/curusarn/resh/pkg/records" +// DirectorySensitive prediction/recommendation strategy type DirectorySensitive struct { history map[string][]string lastPwd string } +// Init see name func (s *DirectorySensitive) Init() { s.history = map[string][]string{} } +// GetTitleAndDescription see name func (s *DirectorySensitive) GetTitleAndDescription() (string, string) { return "directory sensitive (recent)", "Use recent commands executed is the same directory" } +// GetCandidates see name func (s *DirectorySensitive) GetCandidates() []string { return s.history[s.lastPwd] } +// AddHistoryRecord see name func (s *DirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error { // work on history for PWD pwd := record.Pwd @@ -34,6 +39,7 @@ func (s *DirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) er return nil } +// ResetHistory see name func (s *DirectorySensitive) ResetHistory() error { s.Init() s.history = map[string][]string{} diff --git a/pkg/strat/dummy.go b/pkg/strat/dummy.go index b21a60d..fc813f2 100644 --- a/pkg/strat/dummy.go +++ b/pkg/strat/dummy.go @@ -2,23 +2,28 @@ package strat import "github.com/curusarn/resh/pkg/records" +// Dummy prediction/recommendation strategy type Dummy struct { history []string } +// GetTitleAndDescription see name func (s *Dummy) GetTitleAndDescription() (string, string) { return "dummy", "Return empty candidate list" } +// GetCandidates see name func (s *Dummy) GetCandidates() []string { return nil } +// AddHistoryRecord see name func (s *Dummy) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, record.CmdLine) return nil } +// ResetHistory see name func (s *Dummy) ResetHistory() error { return nil } diff --git a/pkg/strat/dynamic-record-distance.go b/pkg/strat/dynamic-record-distance.go index 4a3be10..1f779c2 100644 --- a/pkg/strat/dynamic-record-distance.go +++ b/pkg/strat/dynamic-record-distance.go @@ -8,6 +8,7 @@ import ( "github.com/curusarn/resh/pkg/records" ) +// DynamicRecordDistance prediction/recommendation strategy type DynamicRecordDistance struct { history []records.EnrichedRecord DistParams records.DistParams @@ -23,6 +24,7 @@ type strDynDistEntry struct { distance float64 } +// Init see name func (s *DynamicRecordDistance) Init() { s.history = nil s.pwdHistogram = map[string]int{} @@ -30,6 +32,7 @@ func (s *DynamicRecordDistance) Init() { s.gitOriginHistogram = map[string]int{} } +// GetTitleAndDescription see name func (s *DynamicRecordDistance) GetTitleAndDescription() (string, string) { return "dynamic record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use TF-IDF record distance to recommend commands" } @@ -38,6 +41,7 @@ func (s *DynamicRecordDistance) idf(count int) float64 { return math.Log(float64(len(s.history)) / float64(count)) } +// GetCandidates see name func (s *DynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { if len(s.history) == 0 { return nil @@ -70,6 +74,7 @@ func (s *DynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRec return hist } +// AddHistoryRecord see name func (s *DynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { // append record to front s.history = append([]records.EnrichedRecord{*record}, s.history...) @@ -79,6 +84,7 @@ func (s *DynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) return nil } +// ResetHistory see name func (s *DynamicRecordDistance) ResetHistory() error { s.Init() return nil diff --git a/pkg/strat/frequent.go b/pkg/strat/frequent.go index 6525b9f..ff3b912 100644 --- a/pkg/strat/frequent.go +++ b/pkg/strat/frequent.go @@ -6,6 +6,7 @@ import ( "github.com/curusarn/resh/pkg/records" ) +// Frequent prediction/recommendation strategy type Frequent struct { history map[string]int } @@ -15,14 +16,17 @@ type strFrqEntry struct { count int } -func (s *Frequent) init() { +// Init see name +func (s *Frequent) Init() { s.history = map[string]int{} } +// GetTitleAndDescription see name func (s *Frequent) GetTitleAndDescription() (string, string) { return "frequent", "Use frequent commands" } +// GetCandidates see name func (s *Frequent) GetCandidates() []string { var mapItems []strFrqEntry for cmdLine, count := range s.history { @@ -36,12 +40,14 @@ func (s *Frequent) GetCandidates() []string { return hist } +// AddHistoryRecord see name func (s *Frequent) AddHistoryRecord(record *records.EnrichedRecord) error { s.history[record.CmdLine]++ return nil } +// ResetHistory see name func (s *Frequent) ResetHistory() error { - s.init() + s.Init() return nil } diff --git a/pkg/strat/markov-chain-cmd.go b/pkg/strat/markov-chain-cmd.go index 1bd1ded..b1fa2f5 100644 --- a/pkg/strat/markov-chain-cmd.go +++ b/pkg/strat/markov-chain-cmd.go @@ -8,6 +8,7 @@ import ( "github.com/mb-14/gomarkov" ) +// MarkovChainCmd prediction/recommendation strategy type MarkovChainCmd struct { Order int history []strMarkCmdHistoryEntry @@ -24,15 +25,18 @@ type strMarkCmdEntry struct { transProb float64 } +// Init see name func (s *MarkovChainCmd) Init() { s.history = nil s.historyCmds = nil } +// GetTitleAndDescription see name func (s *MarkovChainCmd) GetTitleAndDescription() (string, string) { return "command-based markov chain (order " + strconv.Itoa(s.Order) + ")", "Use command-based markov chain to recommend commands" } +// GetCandidates see name func (s *MarkovChainCmd) GetCandidates() []string { if len(s.history) < s.Order { var hist []string @@ -78,6 +82,7 @@ func (s *MarkovChainCmd) GetCandidates() []string { return hist } +// AddHistoryRecord see name func (s *MarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command}) s.historyCmds = append(s.historyCmds, record.Command) @@ -85,6 +90,7 @@ func (s *MarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error return nil } +// ResetHistory see name func (s *MarkovChainCmd) ResetHistory() error { s.Init() return nil diff --git a/pkg/strat/markov-chain.go b/pkg/strat/markov-chain.go index 07006e0..50c7fdc 100644 --- a/pkg/strat/markov-chain.go +++ b/pkg/strat/markov-chain.go @@ -8,6 +8,7 @@ import ( "github.com/mb-14/gomarkov" ) +// MarkovChain prediction/recommendation strategy type MarkovChain struct { Order int history []string @@ -18,14 +19,17 @@ type strMarkEntry struct { transProb float64 } +// Init see name func (s *MarkovChain) Init() { s.history = nil } +// GetTitleAndDescription see name func (s *MarkovChain) GetTitleAndDescription() (string, string) { return "markov chain (order " + strconv.Itoa(s.Order) + ")", "Use markov chain to recommend commands" } +// GetCandidates see name func (s *MarkovChain) GetCandidates() []string { if len(s.history) < s.Order { return s.history @@ -58,12 +62,14 @@ func (s *MarkovChain) GetCandidates() []string { return hist } +// AddHistoryRecord see name func (s *MarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append(s.history, record.CmdLine) // s.historySet[record.CmdLine] = true return nil } +// ResetHistory see name func (s *MarkovChain) ResetHistory() error { s.Init() return nil diff --git a/pkg/strat/random.go b/pkg/strat/random.go index 27e959c..0ff52f1 100644 --- a/pkg/strat/random.go +++ b/pkg/strat/random.go @@ -7,21 +7,25 @@ import ( "github.com/curusarn/resh/pkg/records" ) +// Random prediction/recommendation strategy type Random struct { CandidatesSize int history []string historySet map[string]bool } +// Init see name func (s *Random) Init() { s.history = nil s.historySet = map[string]bool{} } +// GetTitleAndDescription see name func (s *Random) GetTitleAndDescription() (string, string) { return "random", "Use random commands" } +// GetCandidates see name func (s *Random) GetCandidates() []string { seed := time.Now().UnixNano() rand.Seed(seed) @@ -39,12 +43,14 @@ func (s *Random) GetCandidates() []string { return candidates } +// AddHistoryRecord see name func (s *Random) AddHistoryRecord(record *records.EnrichedRecord) error { s.history = append([]string{record.CmdLine}, s.history...) s.historySet[record.CmdLine] = true return nil } +// ResetHistory see name func (s *Random) ResetHistory() error { s.Init() return nil diff --git a/pkg/strat/recent-bash.go b/pkg/strat/recent-bash.go index c5319ce..ace3571 100644 --- a/pkg/strat/recent-bash.go +++ b/pkg/strat/recent-bash.go @@ -2,21 +2,25 @@ package strat import "github.com/curusarn/resh/pkg/records" +// RecentBash prediction/recommendation strategy type RecentBash struct { histfile []string histfileSnapshot map[string][]string history map[string][]string } +// Init see name func (s *RecentBash) Init() { s.histfileSnapshot = map[string][]string{} s.history = map[string][]string{} } +// GetTitleAndDescription see name func (s *RecentBash) GetTitleAndDescription() (string, string) { return "recent (bash-like)", "Behave like bash" } +// GetCandidates see name func (s *RecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string { // populate the local history from histfile if s.histfileSnapshot[strippedRecord.SessionID] == nil { @@ -25,6 +29,7 @@ func (s *RecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []stri return append(s.history[strippedRecord.SessionID], s.histfileSnapshot[strippedRecord.SessionID]...) } +// AddHistoryRecord see name func (s *RecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { // remove previous occurance of record for i, cmd := range s.history[record.SessionID] { @@ -44,6 +49,7 @@ func (s *RecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { return nil } +// ResetHistory see name func (s *RecentBash) ResetHistory() error { s.Init() return nil diff --git a/pkg/strat/recent.go b/pkg/strat/recent.go index 68c8eff..157b52c 100644 --- a/pkg/strat/recent.go +++ b/pkg/strat/recent.go @@ -2,18 +2,22 @@ package strat import "github.com/curusarn/resh/pkg/records" +// Recent prediction/recommendation strategy type Recent struct { history []string } +// GetTitleAndDescription see name func (s *Recent) GetTitleAndDescription() (string, string) { return "recent", "Use recent commands" } +// GetCandidates see name func (s *Recent) GetCandidates() []string { return s.history } +// AddHistoryRecord see name func (s *Recent) AddHistoryRecord(record *records.EnrichedRecord) error { // remove previous occurance of record for i, cmd := range s.history { @@ -26,6 +30,7 @@ func (s *Recent) AddHistoryRecord(record *records.EnrichedRecord) error { return nil } +// ResetHistory see name func (s *Recent) ResetHistory() error { s.history = nil return nil diff --git a/pkg/strat/record-distance.go b/pkg/strat/record-distance.go index 816922e..e582584 100644 --- a/pkg/strat/record-distance.go +++ b/pkg/strat/record-distance.go @@ -7,6 +7,7 @@ import ( "github.com/curusarn/resh/pkg/records" ) +// RecordDistance prediction/recommendation strategy type RecordDistance struct { history []records.EnrichedRecord DistParams records.DistParams @@ -19,14 +20,17 @@ type strDistEntry struct { distance float64 } +// Init see name func (s *RecordDistance) Init() { s.history = nil } +// GetTitleAndDescription see name func (s *RecordDistance) GetTitleAndDescription() (string, string) { return "record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use record distance to recommend commands" } +// GetCandidates see name func (s *RecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { if len(s.history) == 0 { return nil @@ -52,12 +56,14 @@ func (s *RecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) [] return hist } +// AddHistoryRecord see name func (s *RecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { // append record to front s.history = append([]records.EnrichedRecord{*record}, s.history...) return nil } +// ResetHistory see name func (s *RecordDistance) ResetHistory() error { s.Init() return nil diff --git a/pkg/strat/strat.go b/pkg/strat/strat.go index d5b1b2c..28ac015 100644 --- a/pkg/strat/strat.go +++ b/pkg/strat/strat.go @@ -4,6 +4,7 @@ import ( "github.com/curusarn/resh/pkg/records" ) +// ISimpleStrategy interface type ISimpleStrategy interface { GetTitleAndDescription() (string, string) GetCandidates() []string @@ -11,6 +12,7 @@ type ISimpleStrategy interface { ResetHistory() error } +// IStrategy interface type IStrategy interface { GetTitleAndDescription() (string, string) GetCandidates(r records.EnrichedRecord) []string From 383216942a86239dbb6a696a403caec47d7d6221 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Sun, 29 Sep 2019 00:53:18 +0200 Subject: [PATCH 11/13] Add tests for pkg/records --- Makefile | 9 +- pkg/histanal/histload.go | 2 +- pkg/records/records.go | 19 ++-- pkg/records/records_test.go | 152 +++++++++++++++++++++++++ pkg/records/testdata/resh_history.json | 27 +++++ 5 files changed, 196 insertions(+), 13 deletions(-) create mode 100644 pkg/records/records_test.go create mode 100644 pkg/records/testdata/resh_history.json diff --git a/Makefile b/Makefile index 83fbbc0..deeb7ee 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,14 @@ sanitize: # # -build: submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize +build: test submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize + +test: + # Running tests + @for dir in {cmd,pkg}/* ; do \ + echo $$dir ; \ + go test $$dir/*.go ; \ + done rebuild: make clean diff --git a/pkg/histanal/histload.go b/pkg/histanal/histload.go index c8f253b..313c7ff 100644 --- a/pkg/histanal/histload.go +++ b/pkg/histanal/histload.go @@ -173,7 +173,7 @@ func (e *HistLoad) loadHistoryRecords(fname string) []records.EnrichedRecord { if record.CmdLength == 0 { log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") } - recs = append(recs, record.Enrich()) + recs = append(recs, records.Enriched(record)) } return recs } diff --git a/pkg/records/records.go b/pkg/records/records.go index 32ea789..f15c717 100644 --- a/pkg/records/records.go +++ b/pkg/records/records.go @@ -127,8 +127,8 @@ func (r EnrichedRecord) ToString() (string, error) { return string(jsonRec), nil } -// Enrich - adds additional fields to the record -func (r Record) Enrich() EnrichedRecord { +// Enriched - returnd enriched record +func Enriched(r Record) EnrichedRecord { record := EnrichedRecord{Record: r} // Get command/first word from commandline var err error @@ -153,9 +153,15 @@ func (r Record) Enrich() EnrichedRecord { // Validate - returns error if the record is invalid func (r *Record) Validate() error { + if r.CmdLine == "" { + return errors.New("There is no CmdLine") + } if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 { return errors.New("There is no Time") } + if r.RealtimeBeforeLocal == 0 || r.RealtimeAfterLocal == 0 { + return errors.New("There is no Local Time") + } if r.RealPwd == "" || r.RealPwdAfter == "" { return errors.New("There is no Real Pwd") } @@ -230,15 +236,6 @@ func Stripped(r EnrichedRecord) EnrichedRecord { return r } -// SetBeforeToAfter - set "before" members to "after" members -func (r *EnrichedRecord) SetBeforeToAfter() { - r.Pwd = r.PwdAfter - r.RealPwd = r.RealPwdAfter - // r.TimezoneBefore = r.TimezoneAfter - // r.RealtimeBefore = r.RealtimeAfter - // r.RealtimeBeforeLocal = r.RealtimeAfterLocal -} - // GetCommandAndFirstWord func func GetCommandAndFirstWord(cmdLine string) (string, string, error) { args, err := shellwords.Parse(cmdLine) diff --git a/pkg/records/records_test.go b/pkg/records/records_test.go new file mode 100644 index 0000000..5ef3c55 --- /dev/null +++ b/pkg/records/records_test.go @@ -0,0 +1,152 @@ +package records + +import ( + "bufio" + "encoding/json" + "log" + "os" + "testing" +) + +func GetTestRecords() []Record { + file, err := os.Open("testdata/resh_history.json") + if err != nil { + log.Fatal("Open() resh history file error:", err) + } + defer file.Close() + + var recs []Record + scanner := bufio.NewScanner(file) + for scanner.Scan() { + record := Record{} + line := scanner.Text() + err = json.Unmarshal([]byte(line), &record) + if err != nil { + log.Println("Line:", line) + log.Fatal("Decoding error:", err) + } + recs = append(recs, record) + } + return recs +} + +func GetTestEnrichedRecords() []EnrichedRecord { + var recs []EnrichedRecord + for _, rec := range GetTestRecords() { + recs = append(recs, Enriched(rec)) + } + return recs +} + +func TestToString(t *testing.T) { + for _, rec := range GetTestEnrichedRecords() { + _, err := rec.ToString() + if err != nil { + t.Error("ToString() failed") + } + } +} + +func TestEnriched(t *testing.T) { + record := Record{BaseRecord: BaseRecord{CmdLine: "cmd arg1 arg2"}} + enriched := Enriched(record) + if enriched.FirstWord != "cmd" || enriched.Command != "cmd" { + t.Error("Enriched() returned reocord w/ wrong Command OR FirstWord") + } +} + +func TestValidate(t *testing.T) { + record := EnrichedRecord{} + if record.Validate() == nil { + t.Error("Validate() didn't return an error for invalid record") + } + record.CmdLine = "cmd arg" + record.FirstWord = "cmd" + record.Command = "cmd" + time := 1234.5678 + record.RealtimeBefore = time + record.RealtimeAfter = time + record.RealtimeBeforeLocal = time + record.RealtimeAfterLocal = time + pwd := "/pwd" + record.Pwd = pwd + record.PwdAfter = pwd + record.RealPwd = pwd + record.RealPwdAfter = pwd + if record.Validate() != nil { + t.Error("Validate() returned an error for a valid record") + } +} + +func TestSetCmdLine(t *testing.T) { + record := EnrichedRecord{} + cmdline := "cmd arg1 arg2" + record.SetCmdLine(cmdline) + if record.CmdLine != cmdline || record.Command != "cmd" || record.FirstWord != "cmd" { + t.Error() + } +} + +func TestStripped(t *testing.T) { + for _, rec := range GetTestEnrichedRecords() { + stripped := Stripped(rec) + + // there should be no cmdline + if stripped.CmdLine != "" || + stripped.FirstWord != "" || + stripped.Command != "" { + t.Error("Stripped() returned record w/ info about CmdLine, Command OR FirstWord") + } + // *after* fields should be overwritten by *before* fields + if stripped.PwdAfter != stripped.Pwd || + stripped.RealPwdAfter != stripped.RealPwd || + stripped.TimezoneAfter != stripped.TimezoneBefore || + stripped.RealtimeAfter != stripped.RealtimeBefore || + stripped.RealtimeAfterLocal != stripped.RealtimeBeforeLocal { + t.Error("Stripped() returned record w/ different *after* and *before* values - *after* fields should be overwritten by *before* fields") + } + // there should be no information about duration and session end + if stripped.RealtimeDuration != 0 || + stripped.LastRecordOfSession != false { + t.Error("Stripped() returned record with too much information") + } + } +} + +func TestGetCommandAndFirstWord(t *testing.T) { + cmd, stWord, err := GetCommandAndFirstWord("cmd arg1 arg2") + if err != nil || cmd != "cmd" || stWord != "cmd" { + t.Error("GetCommandAndFirstWord() returned wrong Command OR FirstWord") + } +} + +func TestDistanceTo(t *testing.T) { + paramsFull := DistParams{ + ExitCode: 1, + MachineID: 1, + SessionID: 1, + Login: 1, + Shell: 1, + Pwd: 1, + RealPwd: 1, + Git: 1, + Time: 1, + } + paramsZero := DistParams{} + var prevRec EnrichedRecord + for _, rec := range GetTestEnrichedRecords() { + dist := rec.DistanceTo(rec, paramsFull) + if dist != 0 { + t.Error("DistanceTo() itself should be always 0") + } + dist = rec.DistanceTo(prevRec, paramsFull) + if dist == 0 { + t.Error("DistanceTo() between two test records shouldn't be 0") + } + dist = rec.DistanceTo(prevRec, paramsZero) + if dist != 0 { + t.Error("DistanceTo() should be 0 when DistParams is all zeros") + } + prevRec = rec + } +} diff --git a/pkg/records/testdata/resh_history.json b/pkg/records/testdata/resh_history.json new file mode 100644 index 0000000..40f43ab --- /dev/null +++ b/pkg/records/testdata/resh_history.json @@ -0,0 +1,27 @@ +{"cmdLine":"ls","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"d5c0fe70-c80b-4715-87cb-f8d8d5b4c673","cols":"80","lines":"24","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14560,"sessionPid":14560,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1566762905.173595,"realtimeAfter":1566762905.1894295,"realtimeBeforeLocal":1566770105.173595,"realtimeAfterLocal":1566770105.1894295,"realtimeDuration":0.015834569931030273,"realtimeSinceSessionStart":1.7122540473937988,"realtimeSinceBoot":20766.542254047396,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"find . -name applications","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567420001.2531302,"realtimeAfter":1567420002.4311218,"realtimeBeforeLocal":1567427201.2531302,"realtimeAfterLocal":1567427202.4311218,"realtimeDuration":1.1779916286468506,"realtimeSinceSessionStart":957.4848053455353,"realtimeSinceBoot":2336.594805345535,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"desktop-file-validate curusarn.sync-clipboards.desktop ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/.local/share/applications","pwdAfter":"/home/simon/.local/share/applications","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/.local/share/applications","realPwdAfter":"/home/simon/.local/share/applications","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567421748.2965438,"realtimeAfter":1567421748.3068867,"realtimeBeforeLocal":1567428948.2965438,"realtimeAfterLocal":1567428948.3068867,"realtimeDuration":0.010342836380004883,"realtimeSinceSessionStart":2704.528218984604,"realtimeSinceBoot":4083.6382189846036,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"cat /tmp/extensions | grep '.'","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461416.6871984,"realtimeAfter":1567461416.7336714,"realtimeBeforeLocal":1567468616.6871984,"realtimeAfterLocal":1567468616.7336714,"realtimeDuration":0.046473026275634766,"realtimeSinceSessionStart":21.45597553253174,"realtimeSinceBoot":43752.03597553253,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"cd git/resh/","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461667.8806899,"realtimeAfter":1567461667.8949044,"realtimeBeforeLocal":1567468867.8806899,"realtimeAfterLocal":1567468867.8949044,"realtimeDuration":0.014214515686035156,"realtimeSinceSessionStart":272.64946699142456,"realtimeSinceBoot":44003.229466991426,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461707.6467602,"realtimeAfter":1567461707.7177293,"realtimeBeforeLocal":1567468907.6467602,"realtimeAfterLocal":1567468907.7177293,"realtimeDuration":0.0709691047668457,"realtimeSinceSessionStart":312.4155373573303,"realtimeSinceBoot":44042.99553735733,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"cat /tmp/extensions | grep '^\\.' | cut -f1 |tr '[:upper:]' '[:lower:]' ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461722.813049,"realtimeAfter":1567461722.8280325,"realtimeBeforeLocal":1567468922.813049,"realtimeAfterLocal":1567468922.8280325,"realtimeDuration":0.014983415603637695,"realtimeSinceSessionStart":327.581826210022,"realtimeSinceBoot":44058.161826210024,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"tig","exitCode":127,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461906.3896828,"realtimeAfter":1567461906.4084594,"realtimeBeforeLocal":1567469106.3896828,"realtimeAfterLocal":1567469106.4084594,"realtimeDuration":0.018776655197143555,"realtimeSinceSessionStart":511.1584599018097,"realtimeSinceBoot":44241.73845990181,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"resh-sanitize-history | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a3318c80-3521-4b22-aa64-ea0f6c641410","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14601,"sessionPid":14601,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567547116.2430356,"realtimeAfter":1567547116.7547352,"realtimeBeforeLocal":1567554316.2430356,"realtimeAfterLocal":1567554316.7547352,"realtimeDuration":0.5116996765136719,"realtimeSinceSessionStart":15.841878414154053,"realtimeSinceBoot":30527.201878414155,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo pacman -S ansible","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609042.0166302,"realtimeAfter":1567609076.9726007,"realtimeBeforeLocal":1567616242.0166302,"realtimeAfterLocal":1567616276.9726007,"realtimeDuration":34.95597052574158,"realtimeSinceSessionStart":1617.0794131755829,"realtimeSinceBoot":6120.029413175583,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"vagrant up","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609090.7359188,"realtimeAfter":1567609098.3125577,"realtimeBeforeLocal":1567616290.7359188,"realtimeAfterLocal":1567616298.3125577,"realtimeDuration":7.57663893699646,"realtimeSinceSessionStart":1665.798701763153,"realtimeSinceBoot":6168.748701763153,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo modprobe vboxnetflt","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609143.2847652,"realtimeAfter":1567609143.3116078,"realtimeBeforeLocal":1567616343.2847652,"realtimeAfterLocal":1567616343.3116078,"realtimeDuration":0.026842594146728516,"realtimeSinceSessionStart":1718.3475482463837,"realtimeSinceBoot":6221.2975482463835,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"echo $RANDOM","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"8ddacadc-6e73-483c-b347-4e18df204466","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":31387,"sessionPid":31387,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567727039.6540458,"realtimeAfter":1567727039.6629689,"realtimeBeforeLocal":1567734239.6540458,"realtimeAfterLocal":1567734239.6629689,"realtimeDuration":0.008923053741455078,"realtimeSinceSessionStart":1470.7667458057404,"realtimeSinceBoot":18495.01674580574,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"make resh-evaluate ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567977478.9672194,"realtimeAfter":1567977479.5449634,"realtimeBeforeLocal":1567984678.9672194,"realtimeAfterLocal":1567984679.5449634,"realtimeDuration":0.5777440071105957,"realtimeSinceSessionStart":5738.577540636063,"realtimeSinceBoot":20980.42754063606,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"cat ~/.resh_history.json | grep \"./resh-eval\" | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"105","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567986105.3988302,"realtimeAfter":1567986105.4809113,"realtimeBeforeLocal":1567993305.3988302,"realtimeAfterLocal":1567993305.4809113,"realtimeDuration":0.08208107948303223,"realtimeSinceSessionStart":14365.00915145874,"realtimeSinceBoot":29606.85915145874,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git c \"add sanitized flag to record, add Enrich() to record\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063976.9103937,"realtimeAfter":1568063976.9326868,"realtimeBeforeLocal":1568071176.9103937,"realtimeAfterLocal":1568071176.9326868,"realtimeDuration":0.0222930908203125,"realtimeSinceSessionStart":92236.52071499825,"realtimeSinceBoot":107478.37071499825,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063978.2340608,"realtimeAfter":1568063978.252463,"realtimeBeforeLocal":1568071178.2340608,"realtimeAfterLocal":1568071178.252463,"realtimeDuration":0.0184023380279541,"realtimeSinceSessionStart":92237.84438204765,"realtimeSinceBoot":107479.69438204766,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git a evaluate/results.go ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063989.0446353,"realtimeAfter":1568063989.2452207,"realtimeBeforeLocal":1568071189.0446353,"realtimeAfterLocal":1568071189.2452207,"realtimeDuration":0.20058536529541016,"realtimeSinceSessionStart":92248.65495657921,"realtimeSinceBoot":107490.50495657921,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo pacman -S python-pip","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072068.3557143,"realtimeAfter":1568072070.7509863,"realtimeBeforeLocal":1568079268.3557143,"realtimeAfterLocal":1568079270.7509863,"realtimeDuration":2.3952720165252686,"realtimeSinceSessionStart":100327.96603560448,"realtimeSinceBoot":115569.81603560448,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"pip3 install matplotlib","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072088.5575967,"realtimeAfter":1568072094.372314,"realtimeBeforeLocal":1568079288.5575967,"realtimeAfterLocal":1568079294.372314,"realtimeDuration":5.8147172927856445,"realtimeSinceSessionStart":100348.16791796684,"realtimeSinceBoot":115590.01791796685,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo pip3 install matplotlib","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072106.138616,"realtimeAfter":1568072115.1124601,"realtimeBeforeLocal":1568079306.138616,"realtimeAfterLocal":1568079315.1124601,"realtimeDuration":8.973844051361084,"realtimeSinceSessionStart":100365.7489373684,"realtimeSinceBoot":115607.5989373684,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"./resh-evaluate --plotting-script evaluate/resh-evaluate-plot.py --input ~/git/resh_private/history_data/simon/dell/resh_history.json ","exitCode":130,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568076266.9364285,"realtimeAfter":1568076288.1131275,"realtimeBeforeLocal":1568083466.9364285,"realtimeAfterLocal":1568083488.1131275,"realtimeDuration":21.176698923110962,"realtimeSinceSessionStart":104526.54674983025,"realtimeSinceBoot":119768.39674983025,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git c \"Add a bunch of useless comments to make linter happy\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"04050353-a97d-4435-9248-f47dd08b2f2a","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":14702,"sessionPid":14702,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569456045.8763022,"realtimeAfter":1569456045.9030173,"realtimeBeforeLocal":1569463245.8763022,"realtimeAfterLocal":1569463245.9030173,"realtimeDuration":0.02671504020690918,"realtimeSinceSessionStart":2289.789242744446,"realtimeSinceBoot":143217.91924274445,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"fuck","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a4aadf03-610d-4731-ba94-5b7ce21e7bb9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":3413,"sessionPid":3413,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569687682.4250975,"realtimeAfter":1569687682.5877323,"realtimeBeforeLocal":1569694882.4250975,"realtimeAfterLocal":1569694882.5877323,"realtimeDuration":0.16263484954833984,"realtimeSinceSessionStart":264603.49496507645,"realtimeSinceBoot":374854.48496507644,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"code .","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709366.523767,"realtimeAfter":1569709367.516908,"realtimeBeforeLocal":1569716566.523767,"realtimeAfterLocal":1569716567.516908,"realtimeDuration":0.9931409358978271,"realtimeSinceSessionStart":23846.908839941025,"realtimeSinceBoot":396539.888839941,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"make test","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709371.89966,"realtimeAfter":1569709377.430194,"realtimeBeforeLocal":1569716571.89966,"realtimeAfterLocal":1569716577.430194,"realtimeDuration":5.530533790588379,"realtimeSinceSessionStart":23852.284733057022,"realtimeSinceBoot":396545.264733057,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"mkdir ~/git/resh/testdata","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"71529b60-2e7b-4d5b-8dc1-6d0740b58e9e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":21224,"sessionPid":21224,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709838.4642656,"realtimeAfter":1569709838.4718792,"realtimeBeforeLocal":1569717038.4642656,"realtimeAfterLocal":1569717038.4718792,"realtimeDuration":0.007613658905029297,"realtimeSinceSessionStart":9.437154054641724,"realtimeSinceBoot":397011.02715405467,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} From 7eeb249722a3c6abc8d17015f7385dc32380a7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Let?= Date: Wed, 2 Oct 2019 19:43:50 +0200 Subject: [PATCH 12/13] Zsh compatibility patch --- scripts/shellrc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/shellrc.sh b/scripts/shellrc.sh index cfb1666..126008b 100644 --- a/scripts/shellrc.sh +++ b/scripts/shellrc.sh @@ -165,7 +165,7 @@ __resh_precmd() { echo "RESH WARNING: You probably just updated RESH - PLEASE RESTART OR RELOAD THIS TERMINAL SESSION (resh revision: $(resh-collect -revision); resh revision of this terminal session: ${__RESH_REVISION})" fi fi - if [ "$__RESH_VERSION" == $(resh-collect -version) ] && [ "$__RESH_REVISION" == $(resh-collect -revision) ]; then + if [ "$__RESH_VERSION" = $(resh-collect -version) ] && [ "$__RESH_REVISION" = $(resh-collect -revision) ]; then resh-collect -requireVersion "$__RESH_VERSION" \ -requireRevision "$__RESH_REVISION" \ -cmdLine "$__RESH_CMDLINE" \ From 0bcf3fd3a9049fdc5574d8868f55ba9c277ce280 Mon Sep 17 00:00:00 2001 From: Simon Let Date: Thu, 3 Oct 2019 01:53:00 +0200 Subject: [PATCH 13/13] Add some tests for shell scripts --- Makefile | 7 +++++-- scripts/test.sh | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100755 scripts/test.sh diff --git a/Makefile b/Makefile index deeb7ee..353f9c1 100644 --- a/Makefile +++ b/Makefile @@ -41,15 +41,18 @@ sanitize: # # -build: test submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize +build: test_go submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize -test: +test_go: # Running tests @for dir in {cmd,pkg}/* ; do \ echo $$dir ; \ go test $$dir/*.go ; \ done +test: + scripts/test.sh + rebuild: make clean make build diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..3eca974 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +[ "${BASH_SOURCE[0]}" != "scripts/test.sh" ] && echo 'Run this script using `make test`' && exit 1 + +for f in scripts/*.sh; do + echo "Running shellcheck on $f ..." + shellcheck $f --shell=bash --severity=error || exit 1 +done + +echo "Checking Zsh syntax of scripts/shellrc.sh ..." +! zsh -n scripts/shellrc.sh && echo "Zsh syntax check failed!" && exit 1 + +for sh in bash zsh; do + echo "Running functions in scripts/shellrc.sh using $sh ..." + ! $sh -c ". scripts/shellrc.sh; __resh_preexec; __resh_precmd" && echo "Error while running functions!" && exit 1 +done + +# TODO: test installation + +exit 0