improve record structure

collect/resh-collect.go

@@ -172,65 +172,67 @@ func main() {
 	}
 
 	rec := common.Record{
-		// core
-		CmdLine:   *cmdLine,
-		ExitCode:  *exitCode,
-		Shell:     *shell,
-		Uname:     *uname,
-		SessionID: *sessionID,
-
 		// posix
 		Cols:  *cols,
 		Lines: *lines,
-
-		Home:  *home,
-		Lang:  *lang,
-		LcAll: *lcAll,
-		Login: *login,
-		// Path:     *path,
-		Pwd:      *pwd,
-		PwdAfter: *pwdAfter,
-		ShellEnv: *shellEnv,
-		Term:     *term,
-
-		// non-posix
-		RealPwd:      realPwd,
-		RealPwdAfter: realPwdAfter,
-		Pid:          *pid,
-		SessionPid:   *sessionPid,
-		Host:         *host,
-		Hosttype:     *hosttype,
-		Ostype:       *ostype,
-		Machtype:     *machtype,
-		Shlvl:        *shlvl,
-
-		// before after
-		TimezoneBefore: *timezoneBefore,
-		TimezoneAfter:  *timezoneAfter,
-
-		RealtimeBefore:      realtimeBefore,
-		RealtimeAfter:       realtimeAfter,
-		RealtimeBeforeLocal: realtimeBeforeLocal,
-		RealtimeAfterLocal:  realtimeAfterLocal,
-
-		RealtimeDuration:          realtimeDuration,
-		RealtimeSinceSessionStart: realtimeSinceSessionStart,
-		RealtimeSinceBoot:         realtimeSinceBoot,
-
-		GitDir:          gitDir,
-		GitRealDir:      gitRealDir,
-		GitOriginRemote: *gitRemote,
-		MachineID:       readFileContent(machineIDPath),
-
-		OsReleaseID:         *osReleaseID,
-		OsReleaseVersionID:  *osReleaseVersionID,
-		OsReleaseIDLike:     *osReleaseIDLike,
-		OsReleaseName:       *osReleaseName,
-		OsReleasePrettyName: *osReleasePrettyName,
-
-		ReshUUID:     readFileContent(reshUUIDPath),
-		ReshVersion:  Version,
-		ReshRevision: Revision,
+		// core
+		BaseRecord: common.BaseRecord{
+			CmdLine:   *cmdLine,
+			ExitCode:  *exitCode,
+			Shell:     *shell,
+			Uname:     *uname,
+			SessionID: *sessionID,
+
+			// posix
+			Home:  *home,
+			Lang:  *lang,
+			LcAll: *lcAll,
+			Login: *login,
+			// Path:     *path,
+			Pwd:      *pwd,
+			PwdAfter: *pwdAfter,
+			ShellEnv: *shellEnv,
+			Term:     *term,
+
+			// non-posix
+			RealPwd:      realPwd,
+			RealPwdAfter: realPwdAfter,
+			Pid:          *pid,
+			SessionPid:   *sessionPid,
+			Host:         *host,
+			Hosttype:     *hosttype,
+			Ostype:       *ostype,
+			Machtype:     *machtype,
+			Shlvl:        *shlvl,
+
+			// before after
+			TimezoneBefore: *timezoneBefore,
+			TimezoneAfter:  *timezoneAfter,
+
+			RealtimeBefore:      realtimeBefore,
+			RealtimeAfter:       realtimeAfter,
+			RealtimeBeforeLocal: realtimeBeforeLocal,
+			RealtimeAfterLocal:  realtimeAfterLocal,
+
+			RealtimeDuration:          realtimeDuration,
+			RealtimeSinceSessionStart: realtimeSinceSessionStart,
+			RealtimeSinceBoot:         realtimeSinceBoot,
+
+			GitDir:          gitDir,
+			GitRealDir:      gitRealDir,
+			GitOriginRemote: *gitRemote,
+			MachineID:       readFileContent(machineIDPath),
+
+			OsReleaseID:         *osReleaseID,
+			OsReleaseVersionID:  *osReleaseVersionID,
+			OsReleaseIDLike:     *osReleaseIDLike,
+			OsReleaseName:       *osReleaseName,
+			OsReleasePrettyName: *osReleasePrettyName,
+
+			ReshUUID:     readFileContent(reshUUIDPath),
+			ReshVersion:  Version,
+			ReshRevision: Revision,
+		},
 	}
 	sendRecord(rec, strconv.Itoa(config.Port))
 }

common/resh-common.go

@@ -7,8 +7,8 @@ import (
 	"github.com/mattn/go-shellwords"
 )
 
-// Record representing single executed command with its metadata
-type Record struct {
+// BaseRecord - common base for Record and FallbackRecord
+type BaseRecord struct {
 	// core
 	CmdLine   string `json:"cmdLine"`
 	ExitCode  int    `json:"exitCode"`
@@ -17,8 +17,6 @@ type Record struct {
 	SessionID string `json:"sessionId"`
 
 	// posix
-	Cols  string `json:"cols"`
-	Lines string `json:"lines"`
 	Home  string `json:"home"`
 	Lang  string `json:"lang"`
 	LcAll string `json:"lcAll"`
@@ -70,156 +68,59 @@ type Record struct {
 	ReshRevision string `json:"reshRevision"`
 
 	// added by sanitizatizer
-	Sanitized bool `json:"sanitized"`
+	Sanitized bool `json:"sanitized,omitempty"`
 	CmdLength int  `json:"cmdLength,omitempty"`
+}
+
+// Record representing single executed command with its metadata
+type Record struct {
+	BaseRecord
+
+	Cols  string `json:"cols"`
+	Lines string `json:"lines"`
+}
+
+// EnrichedRecord - record enriched with additional data
+type EnrichedRecord struct {
+	Record
 
 	// enriching fields - added "later"
-	FirstWord    string `json:"firstWord,omitempty"`
-	Invalid      bool   `json:"invalid,omitempty"`
-	SeqSessionID uint64 `json:"seqSessionID,omitempty"`
+	FirstWord    string `json:"firstWord"`
+	Invalid      bool   `json:"invalid"`
+	SeqSessionID uint64 `json:"seqSessionId"`
+	// SeqSessionID uint64 `json:"seqSessionId,omitempty"`
 }
 
 // FallbackRecord when record is too old and can't be parsed into regular Record
 type FallbackRecord struct {
+	BaseRecord
 	// older version of the record where cols and lines are int
 
-	// core
-	CmdLine   string `json:"cmdLine"`
-	ExitCode  int    `json:"exitCode"`
-	Shell     string `json:"shell"`
-	Uname     string `json:"uname"`
-	SessionID string `json:"sessionId"`
-
-	// posix
-	Cols  int    `json:"cols"`  // notice the in type
-	Lines int    `json:"lines"` // notice the in type
-	Home  string `json:"home"`
-	Lang  string `json:"lang"`
-	LcAll string `json:"lcAll"`
-	Login string `json:"login"`
-	//Path     string `json:"path"`
-	Pwd      string `json:"pwd"`
-	PwdAfter string `json:"pwdAfter"`
-	ShellEnv string `json:"shellEnv"`
-	Term     string `json:"term"`
-
-	// non-posix"`
-	RealPwd      string `json:"realPwd"`
-	RealPwdAfter string `json:"realPwdAfter"`
-	Pid          int    `json:"pid"`
-	SessionPid   int    `json:"sessionPid"`
-	Host         string `json:"host"`
-	Hosttype     string `json:"hosttype"`
-	Ostype       string `json:"ostype"`
-	Machtype     string `json:"machtype"`
-	Shlvl        int    `json:"shlvl"`
-
-	// before after
-	TimezoneBefore string `json:"timezoneBefore"`
-	TimezoneAfter  string `json:"timezoneAfter"`
-
-	RealtimeBefore      float64 `json:"realtimeBefore"`
-	RealtimeAfter       float64 `json:"realtimeAfter"`
-	RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"`
-	RealtimeAfterLocal  float64 `json:"realtimeAfterLocal"`
-
-	RealtimeDuration          float64 `json:"realtimeDuration"`
-	RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"`
-	RealtimeSinceBoot         float64 `json:"realtimeSinceBoot"`
-	//Logs []string      `json: "logs"`
-
-	GitDir          string `json:"gitDir"`
-	GitRealDir      string `json:"gitRealDir"`
-	GitOriginRemote string `json:"gitOriginRemote"`
-	MachineID       string `json:"machineId"`
-
-	OsReleaseID         string `json:"osReleaseId"`
-	OsReleaseVersionID  string `json:"osReleaseVersionId"`
-	OsReleaseIDLike     string `json:"osReleaseIdLike"`
-	OsReleaseName       string `json:"osReleaseName"`
-	OsReleasePrettyName string `json:"osReleasePrettyName"`
-
-	ReshUUID     string `json:"reshUuid"`
-	ReshVersion  string `json:"reshVersion"`
-	ReshRevision string `json:"reshRevision"`
+	Cols  int `json:"cols"`  // notice the int type
+	Lines int `json:"lines"` // notice the int type
 }
 
 // ConvertRecord from FallbackRecord to Record
 func ConvertRecord(r *FallbackRecord) Record {
 	return Record{
-		// core
-		CmdLine:   r.CmdLine,
-		ExitCode:  r.ExitCode,
-		Shell:     r.Shell,
-		Uname:     r.Uname,
-		SessionID: r.SessionID,
-
-		// posix
+		BaseRecord: r.BaseRecord,
 		// these two lines are the only reason we are doing this
 		Cols:  strconv.Itoa(r.Cols),
 		Lines: strconv.Itoa(r.Lines),
-
-		Home:  r.Home,
-		Lang:  r.Lang,
-		LcAll: r.LcAll,
-		Login: r.Login,
-		// Path:     r.path,
-		Pwd:      r.Pwd,
-		PwdAfter: r.PwdAfter,
-		ShellEnv: r.ShellEnv,
-		Term:     r.Term,
-
-		// non-posix
-		RealPwd:      r.RealPwd,
-		RealPwdAfter: r.RealPwdAfter,
-		Pid:          r.Pid,
-		SessionPid:   r.SessionPid,
-		Host:         r.Host,
-		Hosttype:     r.Hosttype,
-		Ostype:       r.Ostype,
-		Machtype:     r.Machtype,
-		Shlvl:        r.Shlvl,
-
-		// before after
-		TimezoneBefore: r.TimezoneBefore,
-		TimezoneAfter:  r.TimezoneAfter,
-
-		RealtimeBefore:      r.RealtimeBefore,
-		RealtimeAfter:       r.RealtimeAfter,
-		RealtimeBeforeLocal: r.RealtimeBeforeLocal,
-		RealtimeAfterLocal:  r.RealtimeAfterLocal,
-
-		RealtimeDuration:          r.RealtimeDuration,
-		RealtimeSinceSessionStart: r.RealtimeSinceSessionStart,
-		RealtimeSinceBoot:         r.RealtimeSinceBoot,
-
-		GitDir:          r.GitDir,
-		GitRealDir:      r.GitRealDir,
-		GitOriginRemote: r.GitOriginRemote,
-		MachineID:       r.MachineID,
-
-		OsReleaseID:         r.OsReleaseID,
-		OsReleaseVersionID:  r.OsReleaseVersionID,
-		OsReleaseIDLike:     r.OsReleaseIDLike,
-		OsReleaseName:       r.OsReleaseName,
-		OsReleasePrettyName: r.OsReleasePrettyName,
-
-		ReshUUID:     r.ReshUUID,
-		ReshVersion:  r.ReshVersion,
-		ReshRevision: r.ReshRevision,
 	}
 }
 
 // Enrich - adds additional fields to the record
-func (r *Record) Enrich() {
+func (r Record) Enrich() EnrichedRecord {
+	record := EnrichedRecord{Record: r}
 	// Get command/first word from commandline
-	r.FirstWord = GetCommandFromCommandLine(r.CmdLine)
+	record.FirstWord = GetCommandFromCommandLine(r.CmdLine)
 	err := r.Validate()
 	if err != nil {
 		log.Println("Invalid command:", r.CmdLine)
-		r.Invalid = true
+		record.Invalid = true
 	}
-	r.Invalid = false
+	return record
 	// TODO: Detect and mark simple commands r.Simple
 }
 

evaluate/resh-evaluate-plot.py

@@ -22,11 +22,11 @@ DATA_records_by_session = defaultdict(list)
 for user in data["UsersRecords"]:
     for device in user["Devices"]:
         for record in device["Records"]:
-            if record["invalid"]:
+            if "invalid" in record and record["invalid"]:
                 continue
             
             DATA_records.append(record)
-            DATA_records_by_session[record["sessionId"]].append(record)
+            DATA_records_by_session[record["seqSessionId"]].append(record)
 
 DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"]))
 
@@ -39,7 +39,6 @@ async_draw = True
 # for strategy in data["Strategies"]:
 #     print(json.dumps(strategy))
 
-
 def zipf(length):
     return list(map(lambda x: 1/2**x, range(0, length)))
 
@@ -265,8 +264,8 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05):
 
         # graphviz sometimes fails - see above
         try:
-            graph.view()
-            # graph.render('/tmp/resh-graphviz-cmdSeq.gv', view=True)
+            # graph.view()
+            graph.render('/tmp/resh-graphviz-cmdSeq-{}.gv'.format(x), view=True)
             break
         except Exception as e:
             trace = traceback.format_exc()

evaluate/resh-evaluate.go

@@ -110,7 +110,7 @@ func main() {
 type strategy interface {
 	GetTitleAndDescription() (string, string)
 	GetCandidates() []string
-	AddHistoryRecord(record *common.Record) error
+	AddHistoryRecord(record *common.EnrichedRecord) error
 	ResetHistory() error
 }
 
@@ -128,7 +128,7 @@ type strategyJSON struct {
 
 type deviceRecords struct {
 	Name    string
-	Records []common.Record
+	Records []common.EnrichedRecord
 }
 
 type userRecords struct {
@@ -184,7 +184,7 @@ func (e *evaluator) processRecords() {
 		for j, device := range e.UsersRecords[i].Devices {
 			sessionIDs := map[string]uint64{}
 			var nextID uint64
-			nextID = 0
+			nextID = 1 // start with 1 because 0 won't get saved to json
 			for k, record := range e.UsersRecords[i].Devices[j].Records {
 				id, found := sessionIDs[record.SessionID]
 				if found == false {
@@ -192,7 +192,7 @@ func (e *evaluator) processRecords() {
 					sessionIDs[record.SessionID] = id
 					nextID++
 				}
-				record.SeqSessionID = id
+				e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id
 				// assert
 				if record.Sanitized != e.sanitizedInput {
 					if e.sanitizedInput {
@@ -200,9 +200,6 @@ func (e *evaluator) processRecords() {
 					}
 					log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present")
 				}
-
-				e.UsersRecords[i].Devices[j].Records[k].Enrich()
-				// device.Records = append(device.Records, record)
 			}
 			sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool {
 				if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
@@ -217,30 +214,34 @@ func (e *evaluator) processRecords() {
 func (e *evaluator) evaluate(strategy strategy) error {
 	title, description := strategy.GetTitleAndDescription()
 	strategyData := strategyJSON{Title: title, Description: description}
-	for _, record := range e.UsersRecords[0].Devices[0].Records {
-		candidates := strategy.GetCandidates()
-
-		matchFound := false
-		for i, candidate := range candidates {
-			// make an option (--calculate-total) to turn this on/off ?
-			// if i >= e.maxCandidates {
-			// 	break
-			// }
-			if candidate == record.CmdLine {
-				match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength}
-				strategyData.Matches = append(strategyData.Matches, match)
-				matchFound = true
-				break
+	for i := range e.UsersRecords {
+		for j := range e.UsersRecords[i].Devices {
+			for _, record := range e.UsersRecords[i].Devices[j].Records {
+				candidates := strategy.GetCandidates()
+
+				matchFound := false
+				for i, candidate := range candidates {
+					// make an option (--calculate-total) to turn this on/off ?
+					// if i >= e.maxCandidates {
+					// 	break
+					// }
+					if candidate == record.CmdLine {
+						match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength}
+						strategyData.Matches = append(strategyData.Matches, match)
+						matchFound = true
+						break
+					}
+				}
+				if matchFound == false {
+					strategyData.Matches = append(strategyData.Matches, matchJSON{})
+				}
+				err := strategy.AddHistoryRecord(&record)
+				if err != nil {
+					log.Println("Error while evauating", err)
+					return err
+				}
 			}
 		}
-		if matchFound == false {
-			strategyData.Matches = append(strategyData.Matches, matchJSON{})
-		}
-		err := strategy.AddHistoryRecord(&record)
-		if err != nil {
-			log.Println("Error while evauating", err)
-			return err
-		}
 	}
 	e.Strategies = append(e.Strategies, strategyData)
 	return nil
@@ -303,14 +304,14 @@ func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath strin
 	return records
 }
 
-func (e *evaluator) loadHistoryRecords(fname string) []common.Record {
+func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord {
 	file, err := os.Open(fname)
 	if err != nil {
 		log.Fatal("Open() resh history file error:", err)
 	}
 	defer file.Close()
 
-	var records []common.Record
+	var records []common.EnrichedRecord
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		record := common.Record{}
@@ -334,7 +335,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.Record {
 		if record.CmdLength == 0 {
 			log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.")
 		}
-		records = append(records, record)
+		records = append(records, record.Enrich())
 	}
 	return records
 }

evaluate/strategy-directory-sensitive.go

@@ -21,7 +21,7 @@ func (s *strategyDirectorySensitive) GetCandidates() []string {
 	return s.history[s.lastPwd]
 }
 
-func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) error {
+func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.EnrichedRecord) error {
 	// work on history for PWD
 	pwd := record.Pwd
 	// remove previous occurance of record

evaluate/strategy-dummy.go

@@ -14,7 +14,7 @@ func (s *strategyDummy) GetCandidates() []string {
 	return nil
 }
 
-func (s *strategyDummy) AddHistoryRecord(record *common.Record) error {
+func (s *strategyDummy) AddHistoryRecord(record *common.EnrichedRecord) error {
 	s.history = append(s.history, record.CmdLine)
 	return nil
 }

evaluate/strategy-frequent.go

@@ -36,7 +36,7 @@ func (s *strategyFrequent) GetCandidates() []string {
 	return hist
 }
 
-func (s *strategyFrequent) AddHistoryRecord(record *common.Record) error {
+func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error {
 	s.history[record.CmdLine]++
 	return nil
 }

evaluate/strategy-recent.go

@@ -14,7 +14,7 @@ func (s *strategyRecent) GetCandidates() []string {
 	return s.history
 }
 
-func (s *strategyRecent) AddHistoryRecord(record *common.Record) error {
+func (s *strategyRecent) AddHistoryRecord(record *common.EnrichedRecord) error {
 	// remove previous occurance of record
 	for i, cmd := range s.history {
 		if cmd == record.CmdLine {