diff --git a/collect/resh-collect.go b/collect/resh-collect.go index e49e918..286faf9 100644 --- a/collect/resh-collect.go +++ b/collect/resh-collect.go @@ -172,65 +172,67 @@ func main() { } rec := common.Record{ - // core - CmdLine: *cmdLine, - ExitCode: *exitCode, - Shell: *shell, - Uname: *uname, - SessionID: *sessionID, - // posix Cols: *cols, Lines: *lines, - - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - Pwd: *pwd, - PwdAfter: *pwdAfter, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - RealPwd: realPwd, - RealPwdAfter: realPwdAfter, - Pid: *pid, - SessionPid: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - TimezoneAfter: *timezoneAfter, - - RealtimeBefore: realtimeBefore, - RealtimeAfter: realtimeAfter, - RealtimeBeforeLocal: realtimeBeforeLocal, - RealtimeAfterLocal: realtimeAfterLocal, - - RealtimeDuration: realtimeDuration, - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - GitDir: gitDir, - GitRealDir: gitRealDir, - GitOriginRemote: *gitRemote, - MachineID: readFileContent(machineIDPath), - - OsReleaseID: *osReleaseID, - OsReleaseVersionID: *osReleaseVersionID, - OsReleaseIDLike: *osReleaseIDLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, - - ReshUUID: readFileContent(reshUUIDPath), - ReshVersion: Version, - ReshRevision: Revision, + // core + BaseRecord: common.BaseRecord{ + CmdLine: *cmdLine, + ExitCode: *exitCode, + Shell: *shell, + Uname: *uname, + SessionID: *sessionID, + + // posix + Home: *home, + Lang: *lang, + LcAll: *lcAll, + Login: *login, + // Path: *path, + Pwd: *pwd, + PwdAfter: *pwdAfter, + ShellEnv: *shellEnv, + Term: *term, + + // non-posix + RealPwd: realPwd, + RealPwdAfter: realPwdAfter, + Pid: *pid, + SessionPid: *sessionPid, + Host: *host, + Hosttype: *hosttype, + Ostype: *ostype, + Machtype: *machtype, + Shlvl: *shlvl, + + // before after + TimezoneBefore: *timezoneBefore, + TimezoneAfter: *timezoneAfter, + + RealtimeBefore: realtimeBefore, + RealtimeAfter: realtimeAfter, + RealtimeBeforeLocal: realtimeBeforeLocal, + RealtimeAfterLocal: realtimeAfterLocal, + + RealtimeDuration: realtimeDuration, + RealtimeSinceSessionStart: realtimeSinceSessionStart, + RealtimeSinceBoot: realtimeSinceBoot, + + GitDir: gitDir, + GitRealDir: gitRealDir, + GitOriginRemote: *gitRemote, + MachineID: readFileContent(machineIDPath), + + OsReleaseID: *osReleaseID, + OsReleaseVersionID: *osReleaseVersionID, + OsReleaseIDLike: *osReleaseIDLike, + OsReleaseName: *osReleaseName, + OsReleasePrettyName: *osReleasePrettyName, + + ReshUUID: readFileContent(reshUUIDPath), + ReshVersion: Version, + ReshRevision: Revision, + }, } sendRecord(rec, strconv.Itoa(config.Port)) } diff --git a/common/resh-common.go b/common/resh-common.go index 1e3189a..908cf9b 100644 --- a/common/resh-common.go +++ b/common/resh-common.go @@ -7,8 +7,8 @@ import ( "github.com/mattn/go-shellwords" ) -// Record representing single executed command with its metadata -type Record struct { +// BaseRecord - common base for Record and FallbackRecord +type BaseRecord struct { // core CmdLine string `json:"cmdLine"` ExitCode int `json:"exitCode"` @@ -17,8 +17,6 @@ type Record struct { SessionID string `json:"sessionId"` // posix - Cols string `json:"cols"` - Lines string `json:"lines"` Home string `json:"home"` Lang string `json:"lang"` LcAll string `json:"lcAll"` @@ -70,156 +68,59 @@ type Record struct { ReshRevision string `json:"reshRevision"` // added by sanitizatizer - Sanitized bool `json:"sanitized"` + Sanitized bool `json:"sanitized,omitempty"` CmdLength int `json:"cmdLength,omitempty"` +} + +// Record representing single executed command with its metadata +type Record struct { + BaseRecord + + Cols string `json:"cols"` + Lines string `json:"lines"` +} + +// EnrichedRecord - record enriched with additional data +type EnrichedRecord struct { + Record // enriching fields - added "later" - FirstWord string `json:"firstWord,omitempty"` - Invalid bool `json:"invalid,omitempty"` - SeqSessionID uint64 `json:"seqSessionID,omitempty"` + FirstWord string `json:"firstWord"` + Invalid bool `json:"invalid"` + SeqSessionID uint64 `json:"seqSessionId"` + // SeqSessionID uint64 `json:"seqSessionId,omitempty"` } // FallbackRecord when record is too old and can't be parsed into regular Record type FallbackRecord struct { + BaseRecord // older version of the record where cols and lines are int - // core - CmdLine string `json:"cmdLine"` - ExitCode int `json:"exitCode"` - Shell string `json:"shell"` - Uname string `json:"uname"` - SessionID string `json:"sessionId"` - - // posix - Cols int `json:"cols"` // notice the in type - Lines int `json:"lines"` // notice the in type - Home string `json:"home"` - Lang string `json:"lang"` - LcAll string `json:"lcAll"` - Login string `json:"login"` - //Path string `json:"path"` - Pwd string `json:"pwd"` - PwdAfter string `json:"pwdAfter"` - ShellEnv string `json:"shellEnv"` - Term string `json:"term"` - - // non-posix"` - RealPwd string `json:"realPwd"` - RealPwdAfter string `json:"realPwdAfter"` - Pid int `json:"pid"` - SessionPid int `json:"sessionPid"` - Host string `json:"host"` - Hosttype string `json:"hosttype"` - Ostype string `json:"ostype"` - Machtype string `json:"machtype"` - Shlvl int `json:"shlvl"` - - // before after - TimezoneBefore string `json:"timezoneBefore"` - TimezoneAfter string `json:"timezoneAfter"` - - RealtimeBefore float64 `json:"realtimeBefore"` - RealtimeAfter float64 `json:"realtimeAfter"` - RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` - RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` - - RealtimeDuration float64 `json:"realtimeDuration"` - RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` - RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` - //Logs []string `json: "logs"` - - GitDir string `json:"gitDir"` - GitRealDir string `json:"gitRealDir"` - GitOriginRemote string `json:"gitOriginRemote"` - MachineID string `json:"machineId"` - - OsReleaseID string `json:"osReleaseId"` - OsReleaseVersionID string `json:"osReleaseVersionId"` - OsReleaseIDLike string `json:"osReleaseIdLike"` - OsReleaseName string `json:"osReleaseName"` - OsReleasePrettyName string `json:"osReleasePrettyName"` - - ReshUUID string `json:"reshUuid"` - ReshVersion string `json:"reshVersion"` - ReshRevision string `json:"reshRevision"` + Cols int `json:"cols"` // notice the int type + Lines int `json:"lines"` // notice the int type } // ConvertRecord from FallbackRecord to Record func ConvertRecord(r *FallbackRecord) Record { return Record{ - // core - CmdLine: r.CmdLine, - ExitCode: r.ExitCode, - Shell: r.Shell, - Uname: r.Uname, - SessionID: r.SessionID, - - // posix + BaseRecord: r.BaseRecord, // these two lines are the only reason we are doing this Cols: strconv.Itoa(r.Cols), Lines: strconv.Itoa(r.Lines), - - Home: r.Home, - Lang: r.Lang, - LcAll: r.LcAll, - Login: r.Login, - // Path: r.path, - Pwd: r.Pwd, - PwdAfter: r.PwdAfter, - ShellEnv: r.ShellEnv, - Term: r.Term, - - // non-posix - RealPwd: r.RealPwd, - RealPwdAfter: r.RealPwdAfter, - Pid: r.Pid, - SessionPid: r.SessionPid, - Host: r.Host, - Hosttype: r.Hosttype, - Ostype: r.Ostype, - Machtype: r.Machtype, - Shlvl: r.Shlvl, - - // before after - TimezoneBefore: r.TimezoneBefore, - TimezoneAfter: r.TimezoneAfter, - - RealtimeBefore: r.RealtimeBefore, - RealtimeAfter: r.RealtimeAfter, - RealtimeBeforeLocal: r.RealtimeBeforeLocal, - RealtimeAfterLocal: r.RealtimeAfterLocal, - - RealtimeDuration: r.RealtimeDuration, - RealtimeSinceSessionStart: r.RealtimeSinceSessionStart, - RealtimeSinceBoot: r.RealtimeSinceBoot, - - GitDir: r.GitDir, - GitRealDir: r.GitRealDir, - GitOriginRemote: r.GitOriginRemote, - MachineID: r.MachineID, - - OsReleaseID: r.OsReleaseID, - OsReleaseVersionID: r.OsReleaseVersionID, - OsReleaseIDLike: r.OsReleaseIDLike, - OsReleaseName: r.OsReleaseName, - OsReleasePrettyName: r.OsReleasePrettyName, - - ReshUUID: r.ReshUUID, - ReshVersion: r.ReshVersion, - ReshRevision: r.ReshRevision, } } // Enrich - adds additional fields to the record -func (r *Record) Enrich() { +func (r Record) Enrich() EnrichedRecord { + record := EnrichedRecord{Record: r} // Get command/first word from commandline - r.FirstWord = GetCommandFromCommandLine(r.CmdLine) + record.FirstWord = GetCommandFromCommandLine(r.CmdLine) err := r.Validate() if err != nil { log.Println("Invalid command:", r.CmdLine) - r.Invalid = true + record.Invalid = true } - r.Invalid = false + return record // TODO: Detect and mark simple commands r.Simple } diff --git a/evaluate/resh-evaluate-plot.py b/evaluate/resh-evaluate-plot.py index 45d9322..2b64485 100755 --- a/evaluate/resh-evaluate-plot.py +++ b/evaluate/resh-evaluate-plot.py @@ -22,11 +22,11 @@ DATA_records_by_session = defaultdict(list) for user in data["UsersRecords"]: for device in user["Devices"]: for record in device["Records"]: - if record["invalid"]: + if "invalid" in record and record["invalid"]: continue DATA_records.append(record) - DATA_records_by_session[record["sessionId"]].append(record) + DATA_records_by_session[record["seqSessionId"]].append(record) DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"])) @@ -39,7 +39,6 @@ async_draw = True # for strategy in data["Strategies"]: # print(json.dumps(strategy)) - def zipf(length): return list(map(lambda x: 1/2**x, range(0, length))) @@ -265,8 +264,8 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05): # graphviz sometimes fails - see above try: - graph.view() - # graph.render('/tmp/resh-graphviz-cmdSeq.gv', view=True) + # graph.view() + graph.render('/tmp/resh-graphviz-cmdSeq-{}.gv'.format(x), view=True) break except Exception as e: trace = traceback.format_exc() diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go index bcde537..53ad49a 100644 --- a/evaluate/resh-evaluate.go +++ b/evaluate/resh-evaluate.go @@ -110,7 +110,7 @@ func main() { type strategy interface { GetTitleAndDescription() (string, string) GetCandidates() []string - AddHistoryRecord(record *common.Record) error + AddHistoryRecord(record *common.EnrichedRecord) error ResetHistory() error } @@ -128,7 +128,7 @@ type strategyJSON struct { type deviceRecords struct { Name string - Records []common.Record + Records []common.EnrichedRecord } type userRecords struct { @@ -184,7 +184,7 @@ func (e *evaluator) processRecords() { for j, device := range e.UsersRecords[i].Devices { sessionIDs := map[string]uint64{} var nextID uint64 - nextID = 0 + nextID = 1 // start with 1 because 0 won't get saved to json for k, record := range e.UsersRecords[i].Devices[j].Records { id, found := sessionIDs[record.SessionID] if found == false { @@ -192,7 +192,7 @@ func (e *evaluator) processRecords() { sessionIDs[record.SessionID] = id nextID++ } - record.SeqSessionID = id + e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id // assert if record.Sanitized != e.sanitizedInput { if e.sanitizedInput { @@ -200,9 +200,6 @@ func (e *evaluator) processRecords() { } log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") } - - e.UsersRecords[i].Devices[j].Records[k].Enrich() - // device.Records = append(device.Records, record) } sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool { if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { @@ -217,30 +214,34 @@ func (e *evaluator) processRecords() { func (e *evaluator) evaluate(strategy strategy) error { title, description := strategy.GetTitleAndDescription() strategyData := strategyJSON{Title: title, Description: description} - for _, record := range e.UsersRecords[0].Devices[0].Records { - candidates := strategy.GetCandidates() - - matchFound := false - for i, candidate := range candidates { - // make an option (--calculate-total) to turn this on/off ? - // if i >= e.maxCandidates { - // break - // } - if candidate == record.CmdLine { - match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} - strategyData.Matches = append(strategyData.Matches, match) - matchFound = true - break + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + for _, record := range e.UsersRecords[i].Devices[j].Records { + candidates := strategy.GetCandidates() + + matchFound := false + for i, candidate := range candidates { + // make an option (--calculate-total) to turn this on/off ? + // if i >= e.maxCandidates { + // break + // } + if candidate == record.CmdLine { + match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} + strategyData.Matches = append(strategyData.Matches, match) + matchFound = true + break + } + } + if matchFound == false { + strategyData.Matches = append(strategyData.Matches, matchJSON{}) + } + err := strategy.AddHistoryRecord(&record) + if err != nil { + log.Println("Error while evauating", err) + return err + } } } - if matchFound == false { - strategyData.Matches = append(strategyData.Matches, matchJSON{}) - } - err := strategy.AddHistoryRecord(&record) - if err != nil { - log.Println("Error while evauating", err) - return err - } } e.Strategies = append(e.Strategies, strategyData) return nil @@ -303,14 +304,14 @@ func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath strin return records } -func (e *evaluator) loadHistoryRecords(fname string) []common.Record { +func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord { file, err := os.Open(fname) if err != nil { log.Fatal("Open() resh history file error:", err) } defer file.Close() - var records []common.Record + var records []common.EnrichedRecord scanner := bufio.NewScanner(file) for scanner.Scan() { record := common.Record{} @@ -334,7 +335,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.Record { if record.CmdLength == 0 { log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") } - records = append(records, record) + records = append(records, record.Enrich()) } return records } diff --git a/evaluate/strategy-directory-sensitive.go b/evaluate/strategy-directory-sensitive.go index 0c00bc4..2915fc3 100644 --- a/evaluate/strategy-directory-sensitive.go +++ b/evaluate/strategy-directory-sensitive.go @@ -21,7 +21,7 @@ func (s *strategyDirectorySensitive) GetCandidates() []string { return s.history[s.lastPwd] } -func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) error { +func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.EnrichedRecord) error { // work on history for PWD pwd := record.Pwd // remove previous occurance of record diff --git a/evaluate/strategy-dummy.go b/evaluate/strategy-dummy.go index 28ed8ec..29f28b7 100644 --- a/evaluate/strategy-dummy.go +++ b/evaluate/strategy-dummy.go @@ -14,7 +14,7 @@ func (s *strategyDummy) GetCandidates() []string { return nil } -func (s *strategyDummy) AddHistoryRecord(record *common.Record) error { +func (s *strategyDummy) AddHistoryRecord(record *common.EnrichedRecord) error { s.history = append(s.history, record.CmdLine) return nil } diff --git a/evaluate/strategy-frequent.go b/evaluate/strategy-frequent.go index c41f852..b88ab91 100644 --- a/evaluate/strategy-frequent.go +++ b/evaluate/strategy-frequent.go @@ -36,7 +36,7 @@ func (s *strategyFrequent) GetCandidates() []string { return hist } -func (s *strategyFrequent) AddHistoryRecord(record *common.Record) error { +func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error { s.history[record.CmdLine]++ return nil } diff --git a/evaluate/strategy-recent.go b/evaluate/strategy-recent.go index 7d24d23..2b6ebd0 100644 --- a/evaluate/strategy-recent.go +++ b/evaluate/strategy-recent.go @@ -14,7 +14,7 @@ func (s *strategyRecent) GetCandidates() []string { return s.history } -func (s *strategyRecent) AddHistoryRecord(record *common.Record) error { +func (s *strategyRecent) AddHistoryRecord(record *common.EnrichedRecord) error { // remove previous occurance of record for i, cmd := range s.history { if cmd == record.CmdLine {