Mirror
/
resh
mirror of https://github.com/curusarn/resh
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve record structure

Browse Source
pull/15/head
Simon Let 6 years ago
parent ff763252f5
commit 4df80d4c67
8 changed files with 128 additions and 225 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 114
      collect/resh-collect.go
  2. 157
      common/resh-common.go
  3. 9
      evaluate/resh-evaluate-plot.py
  4. 65
      evaluate/resh-evaluate.go
  5. 2
      evaluate/strategy-directory-sensitive.go
  6. 2
      evaluate/strategy-dummy.go
  7. 2
      evaluate/strategy-frequent.go
  8. 2
      evaluate/strategy-recent.go

114
collect/resh-collect.go
Unescape View File

@ -172,65 +172,67 @@ func main() {
} }
rec := common.Record{ rec := common.Record{
// core
CmdLine: *cmdLine,
ExitCode: *exitCode,
Shell: *shell,
Uname: *uname,
SessionID: *sessionID,
// posix // posix
Cols: *cols, Cols: *cols,
Lines: *lines, Lines: *lines,
// core
Home: *home, BaseRecord: common.BaseRecord{
Lang: *lang, CmdLine: *cmdLine,
LcAll: *lcAll, ExitCode: *exitCode,
Login: *login, Shell: *shell,
// Path: *path, Uname: *uname,
Pwd: *pwd, SessionID: *sessionID,
PwdAfter: *pwdAfter,
ShellEnv: *shellEnv, // posix
Term: *term, Home: *home,
Lang: *lang,
// non-posix LcAll: *lcAll,
RealPwd: realPwd, Login: *login,
RealPwdAfter: realPwdAfter, // Path: *path,
Pid: *pid, Pwd: *pwd,
SessionPid: *sessionPid, PwdAfter: *pwdAfter,
Host: *host, ShellEnv: *shellEnv,
Hosttype: *hosttype, Term: *term,
Ostype: *ostype,
Machtype: *machtype, // non-posix
Shlvl: *shlvl, RealPwd: realPwd,
RealPwdAfter: realPwdAfter,
// before after Pid: *pid,
TimezoneBefore: *timezoneBefore, SessionPid: *sessionPid,
TimezoneAfter: *timezoneAfter, Host: *host,
Hosttype: *hosttype,
RealtimeBefore: realtimeBefore, Ostype: *ostype,
RealtimeAfter: realtimeAfter, Machtype: *machtype,
RealtimeBeforeLocal: realtimeBeforeLocal, Shlvl: *shlvl,
RealtimeAfterLocal: realtimeAfterLocal,
// before after
RealtimeDuration: realtimeDuration, TimezoneBefore: *timezoneBefore,
RealtimeSinceSessionStart: realtimeSinceSessionStart, TimezoneAfter: *timezoneAfter,
RealtimeSinceBoot: realtimeSinceBoot,
RealtimeBefore: realtimeBefore,
GitDir: gitDir, RealtimeAfter: realtimeAfter,
GitRealDir: gitRealDir, RealtimeBeforeLocal: realtimeBeforeLocal,
GitOriginRemote: *gitRemote, RealtimeAfterLocal: realtimeAfterLocal,
MachineID: readFileContent(machineIDPath),
RealtimeDuration: realtimeDuration,
OsReleaseID: *osReleaseID, RealtimeSinceSessionStart: realtimeSinceSessionStart,
OsReleaseVersionID: *osReleaseVersionID, RealtimeSinceBoot: realtimeSinceBoot,
OsReleaseIDLike: *osReleaseIDLike,
OsReleaseName: *osReleaseName, GitDir: gitDir,
OsReleasePrettyName: *osReleasePrettyName, GitRealDir: gitRealDir,
GitOriginRemote: *gitRemote,
ReshUUID: readFileContent(reshUUIDPath), MachineID: readFileContent(machineIDPath),
ReshVersion: Version,
ReshRevision: Revision, OsReleaseID: *osReleaseID,
OsReleaseVersionID: *osReleaseVersionID,
OsReleaseIDLike: *osReleaseIDLike,
OsReleaseName: *osReleaseName,
OsReleasePrettyName: *osReleasePrettyName,
ReshUUID: readFileContent(reshUUIDPath),
ReshVersion: Version,
ReshRevision: Revision,
},
} }
sendRecord(rec, strconv.Itoa(config.Port)) sendRecord(rec, strconv.Itoa(config.Port))
} }

157
common/resh-common.go
Unescape View File

@ -7,8 +7,8 @@ import (
"github.com/mattn/go-shellwords" "github.com/mattn/go-shellwords"
) )
// Record representing single executed command with its metadata // BaseRecord - common base for Record and FallbackRecord
type Record struct { type BaseRecord struct {
// core // core
CmdLine string `json:"cmdLine"` CmdLine string `json:"cmdLine"`
ExitCode int `json:"exitCode"` ExitCode int `json:"exitCode"`
@ -17,8 +17,6 @@ type Record struct {
SessionID string `json:"sessionId"` SessionID string `json:"sessionId"`
// posix // posix
Cols string `json:"cols"`
Lines string `json:"lines"`
Home string `json:"home"` Home string `json:"home"`
Lang string `json:"lang"` Lang string `json:"lang"`
LcAll string `json:"lcAll"` LcAll string `json:"lcAll"`
@ -70,156 +68,59 @@ type Record struct {
ReshRevision string `json:"reshRevision"` ReshRevision string `json:"reshRevision"`
// added by sanitizatizer // added by sanitizatizer
Sanitized bool `json:"sanitized"` Sanitized bool `json:"sanitized,omitempty"`
CmdLength int `json:"cmdLength,omitempty"` CmdLength int `json:"cmdLength,omitempty"`
}
// Record representing single executed command with its metadata
type Record struct {
BaseRecord
Cols string `json:"cols"`
Lines string `json:"lines"`
}
// EnrichedRecord - record enriched with additional data
type EnrichedRecord struct {
Record
// enriching fields - added "later" // enriching fields - added "later"
FirstWord string `json:"firstWord,omitempty"` FirstWord string `json:"firstWord"`
Invalid bool `json:"invalid,omitempty"` Invalid bool `json:"invalid"`
SeqSessionID uint64 `json:"seqSessionID,omitempty"` SeqSessionID uint64 `json:"seqSessionId"`
// SeqSessionID uint64 `json:"seqSessionId,omitempty"`
} }
// FallbackRecord when record is too old and can't be parsed into regular Record // FallbackRecord when record is too old and can't be parsed into regular Record
type FallbackRecord struct { type FallbackRecord struct {
BaseRecord
// older version of the record where cols and lines are int // older version of the record where cols and lines are int
// core Cols int `json:"cols"` // notice the int type
CmdLine string `json:"cmdLine"` Lines int `json:"lines"` // notice the int type
ExitCode int `json:"exitCode"`
Shell string `json:"shell"`
Uname string `json:"uname"`
SessionID string `json:"sessionId"`
// posix
Cols int `json:"cols"` // notice the in type
Lines int `json:"lines"` // notice the in type
Home string `json:"home"`
Lang string `json:"lang"`
LcAll string `json:"lcAll"`
Login string `json:"login"`
//Path string `json:"path"`
Pwd string `json:"pwd"`
PwdAfter string `json:"pwdAfter"`
ShellEnv string `json:"shellEnv"`
Term string `json:"term"`
// non-posix"`
RealPwd string `json:"realPwd"`
RealPwdAfter string `json:"realPwdAfter"`
Pid int `json:"pid"`
SessionPid int `json:"sessionPid"`
Host string `json:"host"`
Hosttype string `json:"hosttype"`
Ostype string `json:"ostype"`
Machtype string `json:"machtype"`
Shlvl int `json:"shlvl"`
// before after
TimezoneBefore string `json:"timezoneBefore"`
TimezoneAfter string `json:"timezoneAfter"`
RealtimeBefore float64 `json:"realtimeBefore"`
RealtimeAfter float64 `json:"realtimeAfter"`
RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"`
RealtimeAfterLocal float64 `json:"realtimeAfterLocal"`
RealtimeDuration float64 `json:"realtimeDuration"`
RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"`
RealtimeSinceBoot float64 `json:"realtimeSinceBoot"`
//Logs []string `json: "logs"`
GitDir string `json:"gitDir"`
GitRealDir string `json:"gitRealDir"`
GitOriginRemote string `json:"gitOriginRemote"`
MachineID string `json:"machineId"`
OsReleaseID string `json:"osReleaseId"`
OsReleaseVersionID string `json:"osReleaseVersionId"`
OsReleaseIDLike string `json:"osReleaseIdLike"`
OsReleaseName string `json:"osReleaseName"`
OsReleasePrettyName string `json:"osReleasePrettyName"`
ReshUUID string `json:"reshUuid"`
ReshVersion string `json:"reshVersion"`
ReshRevision string `json:"reshRevision"`
} }
// ConvertRecord from FallbackRecord to Record // ConvertRecord from FallbackRecord to Record
func ConvertRecord(r *FallbackRecord) Record { func ConvertRecord(r *FallbackRecord) Record {
return Record{ return Record{
// core BaseRecord: r.BaseRecord,
CmdLine: r.CmdLine,
ExitCode: r.ExitCode,
Shell: r.Shell,
Uname: r.Uname,
SessionID: r.SessionID,
// posix
// these two lines are the only reason we are doing this // these two lines are the only reason we are doing this
Cols: strconv.Itoa(r.Cols), Cols: strconv.Itoa(r.Cols),
Lines: strconv.Itoa(r.Lines), Lines: strconv.Itoa(r.Lines),
Home: r.Home,
Lang: r.Lang,
LcAll: r.LcAll,
Login: r.Login,
// Path: r.path,
Pwd: r.Pwd,
PwdAfter: r.PwdAfter,
ShellEnv: r.ShellEnv,
Term: r.Term,
// non-posix
RealPwd: r.RealPwd,
RealPwdAfter: r.RealPwdAfter,
Pid: r.Pid,
SessionPid: r.SessionPid,
Host: r.Host,
Hosttype: r.Hosttype,
Ostype: r.Ostype,
Machtype: r.Machtype,
Shlvl: r.Shlvl,
// before after
TimezoneBefore: r.TimezoneBefore,
TimezoneAfter: r.TimezoneAfter,
RealtimeBefore: r.RealtimeBefore,
RealtimeAfter: r.RealtimeAfter,
RealtimeBeforeLocal: r.RealtimeBeforeLocal,
RealtimeAfterLocal: r.RealtimeAfterLocal,
RealtimeDuration: r.RealtimeDuration,
RealtimeSinceSessionStart: r.RealtimeSinceSessionStart,
RealtimeSinceBoot: r.RealtimeSinceBoot,
GitDir: r.GitDir,
GitRealDir: r.GitRealDir,
GitOriginRemote: r.GitOriginRemote,
MachineID: r.MachineID,
OsReleaseID: r.OsReleaseID,
OsReleaseVersionID: r.OsReleaseVersionID,
OsReleaseIDLike: r.OsReleaseIDLike,
OsReleaseName: r.OsReleaseName,
OsReleasePrettyName: r.OsReleasePrettyName,
ReshUUID: r.ReshUUID,
ReshVersion: r.ReshVersion,
ReshRevision: r.ReshRevision,
} }
} }
// Enrich - adds additional fields to the record // Enrich - adds additional fields to the record
func (r *Record) Enrich() { func (r Record) Enrich() EnrichedRecord {
record := EnrichedRecord{Record: r}
// Get command/first word from commandline // Get command/first word from commandline
r.FirstWord = GetCommandFromCommandLine(r.CmdLine) record.FirstWord = GetCommandFromCommandLine(r.CmdLine)
err := r.Validate() err := r.Validate()
if err != nil { if err != nil {
log.Println("Invalid command:", r.CmdLine) log.Println("Invalid command:", r.CmdLine)
r.Invalid = true record.Invalid = true
} }
r.Invalid = false return record
// TODO: Detect and mark simple commands r.Simple // TODO: Detect and mark simple commands r.Simple
} }

9
evaluate/resh-evaluate-plot.py
Unescape View File

@ -22,11 +22,11 @@ DATA_records_by_session = defaultdict(list)
for user in data["UsersRecords"]: for user in data["UsersRecords"]:
for device in user["Devices"]: for device in user["Devices"]:
for record in device["Records"]: for record in device["Records"]:
if record["invalid"]: if "invalid" in record and record["invalid"]:
continue continue
DATA_records.append(record) DATA_records.append(record)
DATA_records_by_session[record["sessionId"]].append(record) DATA_records_by_session[record["seqSessionId"]].append(record)
DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"])) DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"]))
@ -39,7 +39,6 @@ async_draw = True
# for strategy in data["Strategies"]: # for strategy in data["Strategies"]:
# print(json.dumps(strategy)) # print(json.dumps(strategy))
def zipf(length): def zipf(length):
return list(map(lambda x: 1/2**x, range(0, length))) return list(map(lambda x: 1/2**x, range(0, length)))
@ -265,8 +264,8 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05):
# graphviz sometimes fails - see above # graphviz sometimes fails - see above
try: try:
graph.view() # graph.view()
# graph.render('/tmp/resh-graphviz-cmdSeq.gv', view=True) graph.render('/tmp/resh-graphviz-cmdSeq-{}.gv'.format(x), view=True)
break break
except Exception as e: except Exception as e:
trace = traceback.format_exc() trace = traceback.format_exc()

65
evaluate/resh-evaluate.go
Unescape View File

@ -110,7 +110,7 @@ func main() {
type strategy interface { type strategy interface {
GetTitleAndDescription() (string, string) GetTitleAndDescription() (string, string)
GetCandidates() []string GetCandidates() []string
AddHistoryRecord(record *common.Record) error AddHistoryRecord(record *common.EnrichedRecord) error
ResetHistory() error ResetHistory() error
} }
@ -128,7 +128,7 @@ type strategyJSON struct {
type deviceRecords struct { type deviceRecords struct {
Name string Name string
Records []common.Record Records []common.EnrichedRecord
} }
type userRecords struct { type userRecords struct {
@ -184,7 +184,7 @@ func (e *evaluator) processRecords() {
for j, device := range e.UsersRecords[i].Devices { for j, device := range e.UsersRecords[i].Devices {
sessionIDs := map[string]uint64{} sessionIDs := map[string]uint64{}
var nextID uint64 var nextID uint64
nextID = 0 nextID = 1 // start with 1 because 0 won't get saved to json
for k, record := range e.UsersRecords[i].Devices[j].Records { for k, record := range e.UsersRecords[i].Devices[j].Records {
id, found := sessionIDs[record.SessionID] id, found := sessionIDs[record.SessionID]
if found == false { if found == false {
@ -192,7 +192,7 @@ func (e *evaluator) processRecords() {
sessionIDs[record.SessionID] = id sessionIDs[record.SessionID] = id
nextID++ nextID++
} }
record.SeqSessionID = id e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id
// assert // assert
if record.Sanitized != e.sanitizedInput { if record.Sanitized != e.sanitizedInput {
if e.sanitizedInput { if e.sanitizedInput {
@ -200,9 +200,6 @@ func (e *evaluator) processRecords() {
} }
log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present")
} }
e.UsersRecords[i].Devices[j].Records[k].Enrich()
// device.Records = append(device.Records, record)
} }
sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool { sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool {
if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
@ -217,30 +214,34 @@ func (e *evaluator) processRecords() {
func (e *evaluator) evaluate(strategy strategy) error { func (e *evaluator) evaluate(strategy strategy) error {
title, description := strategy.GetTitleAndDescription() title, description := strategy.GetTitleAndDescription()
strategyData := strategyJSON{Title: title, Description: description} strategyData := strategyJSON{Title: title, Description: description}
for _, record := range e.UsersRecords[0].Devices[0].Records { for i := range e.UsersRecords {
candidates := strategy.GetCandidates() for j := range e.UsersRecords[i].Devices {
for _, record := range e.UsersRecords[i].Devices[j].Records {
matchFound := false candidates := strategy.GetCandidates()
for i, candidate := range candidates {
// make an option (--calculate-total) to turn this on/off ? matchFound := false
// if i >= e.maxCandidates { for i, candidate := range candidates {
// break // make an option (--calculate-total) to turn this on/off ?
// } // if i >= e.maxCandidates {
if candidate == record.CmdLine { // break
match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} // }
strategyData.Matches = append(strategyData.Matches, match) if candidate == record.CmdLine {
matchFound = true match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength}
break strategyData.Matches = append(strategyData.Matches, match)
matchFound = true
break
}
}
if matchFound == false {
strategyData.Matches = append(strategyData.Matches, matchJSON{})
}
err := strategy.AddHistoryRecord(&record)
if err != nil {
log.Println("Error while evauating", err)
return err
}
} }
} }
if matchFound == false {
strategyData.Matches = append(strategyData.Matches, matchJSON{})
}
err := strategy.AddHistoryRecord(&record)
if err != nil {
log.Println("Error while evauating", err)
return err
}
} }
e.Strategies = append(e.Strategies, strategyData) e.Strategies = append(e.Strategies, strategyData)
return nil return nil
@ -303,14 +304,14 @@ func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath strin
return records return records
} }
func (e *evaluator) loadHistoryRecords(fname string) []common.Record { func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord {
file, err := os.Open(fname) file, err := os.Open(fname)
if err != nil { if err != nil {
log.Fatal("Open() resh history file error:", err) log.Fatal("Open() resh history file error:", err)
} }
defer file.Close() defer file.Close()
var records []common.Record var records []common.EnrichedRecord
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
for scanner.Scan() { for scanner.Scan() {
record := common.Record{} record := common.Record{}
@ -334,7 +335,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.Record {
if record.CmdLength == 0 { if record.CmdLength == 0 {
log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.")
} }
records = append(records, record) records = append(records, record.Enrich())
} }
return records return records
} }

2
evaluate/strategy-directory-sensitive.go
Unescape View File

@ -21,7 +21,7 @@ func (s *strategyDirectorySensitive) GetCandidates() []string {
return s.history[s.lastPwd] return s.history[s.lastPwd]
} }
func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) error { func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.EnrichedRecord) error {
// work on history for PWD // work on history for PWD
pwd := record.Pwd pwd := record.Pwd
// remove previous occurance of record // remove previous occurance of record

2
evaluate/strategy-dummy.go
Unescape View File

@ -14,7 +14,7 @@ func (s *strategyDummy) GetCandidates() []string {
return nil return nil
} }
func (s *strategyDummy) AddHistoryRecord(record *common.Record) error { func (s *strategyDummy) AddHistoryRecord(record *common.EnrichedRecord) error {
s.history = append(s.history, record.CmdLine) s.history = append(s.history, record.CmdLine)
return nil return nil
} }

2
evaluate/strategy-frequent.go
Unescape View File

@ -36,7 +36,7 @@ func (s *strategyFrequent) GetCandidates() []string {
return hist return hist
} }
func (s *strategyFrequent) AddHistoryRecord(record *common.Record) error { func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error {
s.history[record.CmdLine]++ s.history[record.CmdLine]++
return nil return nil
} }

2
evaluate/strategy-recent.go
Unescape View File

@ -14,7 +14,7 @@ func (s *strategyRecent) GetCandidates() []string {
return s.history return s.history
} }
func (s *strategyRecent) AddHistoryRecord(record *common.Record) error { func (s *strategyRecent) AddHistoryRecord(record *common.EnrichedRecord) error {
// remove previous occurance of record // remove previous occurance of record
for i, cmd := range s.history { for i, cmd := range s.history {
if cmd == record.CmdLine { if cmd == record.CmdLine {

Write Preview
Loading…
Cancel
Save