Mirror
/
resh
mirror of https://github.com/curusarn/resh
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve record structure

Browse Source
pull/15/head
Simon Let 6 years ago
parent ff763252f5
commit 4df80d4c67
8 changed files with 128 additions and 225 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 8
      collect/resh-collect.go
  2. 157
      common/resh-common.go
  3. 9
      evaluate/resh-evaluate-plot.py
  4. 23
      evaluate/resh-evaluate.go
  5. 2
      evaluate/strategy-directory-sensitive.go
  6. 2
      evaluate/strategy-dummy.go
  7. 2
      evaluate/strategy-frequent.go
  8. 2
      evaluate/strategy-recent.go

8
collect/resh-collect.go
Unescape View File

@ -172,7 +172,11 @@ func main() {
}
rec := common.Record{
// posix
Cols: *cols,
Lines: *lines,
// core
BaseRecord: common.BaseRecord{
CmdLine: *cmdLine,
ExitCode: *exitCode,
Shell: *shell,
@ -180,9 +184,6 @@ func main() {
SessionID: *sessionID,
// posix
Cols: *cols,
Lines: *lines,
Home: *home,
Lang: *lang,
LcAll: *lcAll,
@ -231,6 +232,7 @@ func main() {
ReshUUID: readFileContent(reshUUIDPath),
ReshVersion: Version,
ReshRevision: Revision,
},
}
sendRecord(rec, strconv.Itoa(config.Port))
}

157
common/resh-common.go
Unescape View File

@ -7,8 +7,8 @@ import (
"github.com/mattn/go-shellwords"
)
// Record representing single executed command with its metadata
type Record struct {
// BaseRecord - common base for Record and FallbackRecord
type BaseRecord struct {
// core
CmdLine string `json:"cmdLine"`
ExitCode int `json:"exitCode"`
@ -17,8 +17,6 @@ type Record struct {
SessionID string `json:"sessionId"`
// posix
Cols string `json:"cols"`
Lines string `json:"lines"`
Home string `json:"home"`
Lang string `json:"lang"`
LcAll string `json:"lcAll"`
@ -70,156 +68,59 @@ type Record struct {
ReshRevision string `json:"reshRevision"`
// added by sanitizatizer
Sanitized bool `json:"sanitized"`
Sanitized bool `json:"sanitized,omitempty"`
CmdLength int `json:"cmdLength,omitempty"`
}
// Record representing single executed command with its metadata
type Record struct {
BaseRecord
Cols string `json:"cols"`
Lines string `json:"lines"`
}
// EnrichedRecord - record enriched with additional data
type EnrichedRecord struct {
Record
// enriching fields - added "later"
FirstWord string `json:"firstWord,omitempty"`
Invalid bool `json:"invalid,omitempty"`
SeqSessionID uint64 `json:"seqSessionID,omitempty"`
FirstWord string `json:"firstWord"`
Invalid bool `json:"invalid"`
SeqSessionID uint64 `json:"seqSessionId"`
// SeqSessionID uint64 `json:"seqSessionId,omitempty"`
}
// FallbackRecord when record is too old and can't be parsed into regular Record
type FallbackRecord struct {
BaseRecord
// older version of the record where cols and lines are int
// core
CmdLine string `json:"cmdLine"`
ExitCode int `json:"exitCode"`
Shell string `json:"shell"`
Uname string `json:"uname"`
SessionID string `json:"sessionId"`
// posix
Cols int `json:"cols"` // notice the in type
Lines int `json:"lines"` // notice the in type
Home string `json:"home"`
Lang string `json:"lang"`
LcAll string `json:"lcAll"`
Login string `json:"login"`
//Path string `json:"path"`
Pwd string `json:"pwd"`
PwdAfter string `json:"pwdAfter"`
ShellEnv string `json:"shellEnv"`
Term string `json:"term"`
// non-posix"`
RealPwd string `json:"realPwd"`
RealPwdAfter string `json:"realPwdAfter"`
Pid int `json:"pid"`
SessionPid int `json:"sessionPid"`
Host string `json:"host"`
Hosttype string `json:"hosttype"`
Ostype string `json:"ostype"`
Machtype string `json:"machtype"`
Shlvl int `json:"shlvl"`
// before after
TimezoneBefore string `json:"timezoneBefore"`
TimezoneAfter string `json:"timezoneAfter"`
RealtimeBefore float64 `json:"realtimeBefore"`
RealtimeAfter float64 `json:"realtimeAfter"`
RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"`
RealtimeAfterLocal float64 `json:"realtimeAfterLocal"`
RealtimeDuration float64 `json:"realtimeDuration"`
RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"`
RealtimeSinceBoot float64 `json:"realtimeSinceBoot"`
//Logs []string `json: "logs"`
GitDir string `json:"gitDir"`
GitRealDir string `json:"gitRealDir"`
GitOriginRemote string `json:"gitOriginRemote"`
MachineID string `json:"machineId"`
OsReleaseID string `json:"osReleaseId"`
OsReleaseVersionID string `json:"osReleaseVersionId"`
OsReleaseIDLike string `json:"osReleaseIdLike"`
OsReleaseName string `json:"osReleaseName"`
OsReleasePrettyName string `json:"osReleasePrettyName"`
ReshUUID string `json:"reshUuid"`
ReshVersion string `json:"reshVersion"`
ReshRevision string `json:"reshRevision"`
Cols int `json:"cols"` // notice the int type
Lines int `json:"lines"` // notice the int type
}
// ConvertRecord from FallbackRecord to Record
func ConvertRecord(r *FallbackRecord) Record {
return Record{
// core
CmdLine: r.CmdLine,
ExitCode: r.ExitCode,
Shell: r.Shell,
Uname: r.Uname,
SessionID: r.SessionID,
// posix
BaseRecord: r.BaseRecord,
// these two lines are the only reason we are doing this
Cols: strconv.Itoa(r.Cols),
Lines: strconv.Itoa(r.Lines),
Home: r.Home,
Lang: r.Lang,
LcAll: r.LcAll,
Login: r.Login,
// Path: r.path,
Pwd: r.Pwd,
PwdAfter: r.PwdAfter,
ShellEnv: r.ShellEnv,
Term: r.Term,
// non-posix
RealPwd: r.RealPwd,
RealPwdAfter: r.RealPwdAfter,
Pid: r.Pid,
SessionPid: r.SessionPid,
Host: r.Host,
Hosttype: r.Hosttype,
Ostype: r.Ostype,
Machtype: r.Machtype,
Shlvl: r.Shlvl,
// before after
TimezoneBefore: r.TimezoneBefore,
TimezoneAfter: r.TimezoneAfter,
RealtimeBefore: r.RealtimeBefore,
RealtimeAfter: r.RealtimeAfter,
RealtimeBeforeLocal: r.RealtimeBeforeLocal,
RealtimeAfterLocal: r.RealtimeAfterLocal,
RealtimeDuration: r.RealtimeDuration,
RealtimeSinceSessionStart: r.RealtimeSinceSessionStart,
RealtimeSinceBoot: r.RealtimeSinceBoot,
GitDir: r.GitDir,
GitRealDir: r.GitRealDir,
GitOriginRemote: r.GitOriginRemote,
MachineID: r.MachineID,
OsReleaseID: r.OsReleaseID,
OsReleaseVersionID: r.OsReleaseVersionID,
OsReleaseIDLike: r.OsReleaseIDLike,
OsReleaseName: r.OsReleaseName,
OsReleasePrettyName: r.OsReleasePrettyName,
ReshUUID: r.ReshUUID,
ReshVersion: r.ReshVersion,
ReshRevision: r.ReshRevision,
}
}
// Enrich - adds additional fields to the record
func (r *Record) Enrich() {
func (r Record) Enrich() EnrichedRecord {
record := EnrichedRecord{Record: r}
// Get command/first word from commandline
r.FirstWord = GetCommandFromCommandLine(r.CmdLine)
record.FirstWord = GetCommandFromCommandLine(r.CmdLine)
err := r.Validate()
if err != nil {
log.Println("Invalid command:", r.CmdLine)
r.Invalid = true
record.Invalid = true
}
r.Invalid = false
return record
// TODO: Detect and mark simple commands r.Simple
}

9
evaluate/resh-evaluate-plot.py
Unescape View File

@ -22,11 +22,11 @@ DATA_records_by_session = defaultdict(list)
for user in data["UsersRecords"]:
for device in user["Devices"]:
for record in device["Records"]:
if record["invalid"]:
if "invalid" in record and record["invalid"]:
continue
DATA_records.append(record)
DATA_records_by_session[record["sessionId"]].append(record)
DATA_records_by_session[record["seqSessionId"]].append(record)
DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"]))
@ -39,7 +39,6 @@ async_draw = True
# for strategy in data["Strategies"]:
# print(json.dumps(strategy))
def zipf(length):
return list(map(lambda x: 1/2**x, range(0, length)))
@ -265,8 +264,8 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05):
# graphviz sometimes fails - see above
try:
graph.view()
# graph.render('/tmp/resh-graphviz-cmdSeq.gv', view=True)
# graph.view()
graph.render('/tmp/resh-graphviz-cmdSeq-{}.gv'.format(x), view=True)
break
except Exception as e:
trace = traceback.format_exc()

23
evaluate/resh-evaluate.go
Unescape View File

@ -110,7 +110,7 @@ func main() {
type strategy interface {
GetTitleAndDescription() (string, string)
GetCandidates() []string
AddHistoryRecord(record *common.Record) error
AddHistoryRecord(record *common.EnrichedRecord) error
ResetHistory() error
}
@ -128,7 +128,7 @@ type strategyJSON struct {
type deviceRecords struct {
Name string
Records []common.Record
Records []common.EnrichedRecord
}
type userRecords struct {
@ -184,7 +184,7 @@ func (e *evaluator) processRecords() {
for j, device := range e.UsersRecords[i].Devices {
sessionIDs := map[string]uint64{}
var nextID uint64
nextID = 0
nextID = 1 // start with 1 because 0 won't get saved to json
for k, record := range e.UsersRecords[i].Devices[j].Records {
id, found := sessionIDs[record.SessionID]
if found == false {
@ -192,7 +192,7 @@ func (e *evaluator) processRecords() {
sessionIDs[record.SessionID] = id
nextID++
}
record.SeqSessionID = id
e.UsersRecords[i].Devices[j].Records[k].SeqSessionID = id
// assert
if record.Sanitized != e.sanitizedInput {
if e.sanitizedInput {
@ -200,9 +200,6 @@ func (e *evaluator) processRecords() {
}
log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present")
}
e.UsersRecords[i].Devices[j].Records[k].Enrich()
// device.Records = append(device.Records, record)
}
sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool {
if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
@ -217,7 +214,9 @@ func (e *evaluator) processRecords() {
func (e *evaluator) evaluate(strategy strategy) error {
title, description := strategy.GetTitleAndDescription()
strategyData := strategyJSON{Title: title, Description: description}
for _, record := range e.UsersRecords[0].Devices[0].Records {
for i := range e.UsersRecords {
for j := range e.UsersRecords[i].Devices {
for _, record := range e.UsersRecords[i].Devices[j].Records {
candidates := strategy.GetCandidates()
matchFound := false
@ -242,6 +241,8 @@ func (e *evaluator) evaluate(strategy strategy) error {
return err
}
}
}
}
e.Strategies = append(e.Strategies, strategyData)
return nil
}
@ -303,14 +304,14 @@ func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath strin
return records
}
func (e *evaluator) loadHistoryRecords(fname string) []common.Record {
func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord {
file, err := os.Open(fname)
if err != nil {
log.Fatal("Open() resh history file error:", err)
}
defer file.Close()
var records []common.Record
var records []common.EnrichedRecord
scanner := bufio.NewScanner(file)
for scanner.Scan() {
record := common.Record{}
@ -334,7 +335,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.Record {
if record.CmdLength == 0 {
log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.")
}
records = append(records, record)
records = append(records, record.Enrich())
}
return records
}

2
evaluate/strategy-directory-sensitive.go
Unescape View File

@ -21,7 +21,7 @@ func (s *strategyDirectorySensitive) GetCandidates() []string {
return s.history[s.lastPwd]
}
func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) error {
func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.EnrichedRecord) error {
// work on history for PWD
pwd := record.Pwd
// remove previous occurance of record

2
evaluate/strategy-dummy.go
Unescape View File

@ -14,7 +14,7 @@ func (s *strategyDummy) GetCandidates() []string {
return nil
}
func (s *strategyDummy) AddHistoryRecord(record *common.Record) error {
func (s *strategyDummy) AddHistoryRecord(record *common.EnrichedRecord) error {
s.history = append(s.history, record.CmdLine)
return nil
}

2
evaluate/strategy-frequent.go
Unescape View File

@ -36,7 +36,7 @@ func (s *strategyFrequent) GetCandidates() []string {
return hist
}
func (s *strategyFrequent) AddHistoryRecord(record *common.Record) error {
func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error {
s.history[record.CmdLine]++
return nil
}

2
evaluate/strategy-recent.go
Unescape View File

@ -14,7 +14,7 @@ func (s *strategyRecent) GetCandidates() []string {
return s.history
}
func (s *strategyRecent) AddHistoryRecord(record *common.Record) error {
func (s *strategyRecent) AddHistoryRecord(record *common.EnrichedRecord) error {
// remove previous occurance of record
for i, cmd := range s.history {
if cmd == record.CmdLine {

Write Preview
Loading…
Cancel
Save