mirror of https://github.com/curusarn/resh
Simon Let
6 years ago
parent
96cf2ae032
commit
79d7f1c45e
@ -1,24 +0,0 @@ |
||||
package main |
||||
|
||||
import "github.com/curusarn/resh/pkg/records" |
||||
|
||||
type strategyDummy struct { |
||||
history []string |
||||
} |
||||
|
||||
func (s *strategyDummy) GetTitleAndDescription() (string, string) { |
||||
return "dummy", "Return empty candidate list" |
||||
} |
||||
|
||||
func (s *strategyDummy) GetCandidates() []string { |
||||
return nil |
||||
} |
||||
|
||||
func (s *strategyDummy) AddHistoryRecord(record *records.EnrichedRecord) error { |
||||
s.history = append(s.history, record.CmdLine) |
||||
return nil |
||||
} |
||||
|
||||
func (s *strategyDummy) ResetHistory() error { |
||||
return nil |
||||
} |
||||
@ -0,0 +1,246 @@ |
||||
package histanal |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"log" |
||||
"math/rand" |
||||
"os" |
||||
"os/exec" |
||||
|
||||
"github.com/curusarn/resh/pkg/records" |
||||
"github.com/curusarn/resh/pkg/strat" |
||||
"github.com/jpillora/longestcommon" |
||||
|
||||
"github.com/schollz/progressbar" |
||||
) |
||||
|
||||
type matchJSON struct { |
||||
Match bool |
||||
Distance int |
||||
CharsRecalled int |
||||
} |
||||
|
||||
type multiMatchItemJSON struct { |
||||
Distance int |
||||
CharsRecalled int |
||||
} |
||||
|
||||
type multiMatchJSON struct { |
||||
Match bool |
||||
Entries []multiMatchItemJSON |
||||
} |
||||
|
||||
type strategyJSON struct { |
||||
Title string |
||||
Description string |
||||
Matches []matchJSON |
||||
PrefixMatches []multiMatchJSON |
||||
} |
||||
|
||||
// HistEval evaluates history
|
||||
type HistEval struct { |
||||
HistLoad |
||||
BatchMode bool |
||||
maxCandidates int |
||||
Strategies []strategyJSON |
||||
} |
||||
|
||||
// NewHistEval constructs new HistEval
|
||||
func NewHistEval(inputPath string, |
||||
maxCandidates int, skipFailedCmds bool, |
||||
debugRecords float64, sanitizedInput bool) HistEval { |
||||
|
||||
e := HistEval{ |
||||
HistLoad: HistLoad{ |
||||
skipFailedCmds: skipFailedCmds, |
||||
debugRecords: debugRecords, |
||||
sanitizedInput: sanitizedInput, |
||||
}, |
||||
maxCandidates: maxCandidates, |
||||
BatchMode: false, |
||||
} |
||||
records := e.loadHistoryRecords(inputPath) |
||||
device := deviceRecords{Records: records} |
||||
user := userRecords{} |
||||
user.Devices = append(user.Devices, device) |
||||
e.UsersRecords = append(e.UsersRecords, user) |
||||
e.preprocessRecords() |
||||
return e |
||||
} |
||||
|
||||
// NewHistEvalBatchMode constructs new HistEval in batch mode
|
||||
func NewHistEvalBatchMode(input string, inputDataRoot string, |
||||
maxCandidates int, skipFailedCmds bool, |
||||
debugRecords float64, sanitizedInput bool) HistEval { |
||||
|
||||
e := HistEval{ |
||||
HistLoad: HistLoad{ |
||||
skipFailedCmds: skipFailedCmds, |
||||
debugRecords: debugRecords, |
||||
sanitizedInput: sanitizedInput, |
||||
}, |
||||
maxCandidates: maxCandidates, |
||||
BatchMode: false, |
||||
} |
||||
e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) |
||||
e.preprocessRecords() |
||||
return e |
||||
} |
||||
|
||||
func (e *HistEval) preprocessDeviceRecords(device deviceRecords) deviceRecords { |
||||
sessionIDs := map[string]uint64{} |
||||
var nextID uint64 |
||||
nextID = 1 // start with 1 because 0 won't get saved to json
|
||||
for k, record := range device.Records { |
||||
id, found := sessionIDs[record.SessionID] |
||||
if found == false { |
||||
id = nextID |
||||
sessionIDs[record.SessionID] = id |
||||
nextID++ |
||||
} |
||||
device.Records[k].SeqSessionID = id |
||||
// assert
|
||||
if record.Sanitized != e.sanitizedInput { |
||||
if e.sanitizedInput { |
||||
log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") |
||||
} |
||||
log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") |
||||
} |
||||
device.Records[k].SeqSessionID = id |
||||
if e.debugRecords > 0 && rand.Float64() < e.debugRecords { |
||||
device.Records[k].DebugThisRecord = true |
||||
} |
||||
} |
||||
// sort.SliceStable(device.Records, func(x, y int) bool {
|
||||
// if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
|
||||
// return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal
|
||||
// }
|
||||
// return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID
|
||||
// })
|
||||
|
||||
// iterate from back and mark last record of each session
|
||||
sessionIDSet := map[string]bool{} |
||||
for i := len(device.Records) - 1; i >= 0; i-- { |
||||
var record *records.EnrichedRecord |
||||
record = &device.Records[i] |
||||
if sessionIDSet[record.SessionID] { |
||||
continue |
||||
} |
||||
sessionIDSet[record.SessionID] = true |
||||
record.LastRecordOfSession = true |
||||
} |
||||
return device |
||||
} |
||||
|
||||
// enrich records and add sequential session ID
|
||||
func (e *HistEval) preprocessRecords() { |
||||
for i := range e.UsersRecords { |
||||
for j := range e.UsersRecords[i].Devices { |
||||
e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Evaluate a given strategy
|
||||
func (e *HistEval) Evaluate(strategy strat.IStrategy) error { |
||||
title, description := strategy.GetTitleAndDescription() |
||||
log.Println("Evaluating strategy:", title, "-", description) |
||||
strategyData := strategyJSON{Title: title, Description: description} |
||||
for i := range e.UsersRecords { |
||||
for j := range e.UsersRecords[i].Devices { |
||||
bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) |
||||
var prevRecord records.EnrichedRecord |
||||
for _, record := range e.UsersRecords[i].Devices[j].Records { |
||||
if e.skipFailedCmds && record.ExitCode != 0 { |
||||
continue |
||||
} |
||||
candidates := strategy.GetCandidates(records.Stripped(record)) |
||||
if record.DebugThisRecord { |
||||
log.Println() |
||||
log.Println("===================================================") |
||||
log.Println("STRATEGY:", title, "-", description) |
||||
log.Println("===================================================") |
||||
log.Println("Previous record:") |
||||
if prevRecord.RealtimeBefore == 0 { |
||||
log.Println("== NIL") |
||||
} else { |
||||
rec, _ := prevRecord.ToString() |
||||
log.Println(rec) |
||||
} |
||||
log.Println("---------------------------------------------------") |
||||
log.Println("Recommendations for:") |
||||
rec, _ := record.ToString() |
||||
log.Println(rec) |
||||
log.Println("---------------------------------------------------") |
||||
for i, candidate := range candidates { |
||||
if i > 10 { |
||||
break |
||||
} |
||||
log.Println(string(candidate)) |
||||
} |
||||
log.Println("===================================================") |
||||
} |
||||
|
||||
matchFound := false |
||||
longestPrefixMatchLength := 0 |
||||
multiMatch := multiMatchJSON{} |
||||
for i, candidate := range candidates { |
||||
// make an option (--calculate-total) to turn this on/off ?
|
||||
// if i >= e.maxCandidates {
|
||||
// break
|
||||
// }
|
||||
commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine})) |
||||
if commonPrefixLength > longestPrefixMatchLength { |
||||
longestPrefixMatchLength = commonPrefixLength |
||||
prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength} |
||||
multiMatch.Match = true |
||||
multiMatch.Entries = append(multiMatch.Entries, prefixMatch) |
||||
} |
||||
if candidate == record.CmdLine { |
||||
match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} |
||||
matchFound = true |
||||
strategyData.Matches = append(strategyData.Matches, match) |
||||
strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) |
||||
break |
||||
} |
||||
} |
||||
if matchFound == false { |
||||
strategyData.Matches = append(strategyData.Matches, matchJSON{}) |
||||
strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) |
||||
} |
||||
err := strategy.AddHistoryRecord(&record) |
||||
if err != nil { |
||||
log.Println("Error while evauating", err) |
||||
return err |
||||
} |
||||
bar.Add(1) |
||||
prevRecord = record |
||||
} |
||||
strategy.ResetHistory() |
||||
fmt.Println() |
||||
} |
||||
} |
||||
e.Strategies = append(e.Strategies, strategyData) |
||||
return nil |
||||
} |
||||
|
||||
// CalculateStatsAndPlot results
|
||||
func (e *HistEval) CalculateStatsAndPlot(scriptName string) { |
||||
evalJSON, err := json.Marshal(e) |
||||
if err != nil { |
||||
log.Fatal("json marshal error", err) |
||||
} |
||||
buffer := bytes.Buffer{} |
||||
buffer.Write(evalJSON) |
||||
// run python script to stat and plot/
|
||||
cmd := exec.Command(scriptName) |
||||
cmd.Stdout = os.Stdout |
||||
cmd.Stderr = os.Stderr |
||||
cmd.Stdin = &buffer |
||||
err = cmd.Run() |
||||
if err != nil { |
||||
log.Printf("Command finished with error: %v", err) |
||||
} |
||||
} |
||||
@ -0,0 +1,179 @@ |
||||
package histanal |
||||
|
||||
import ( |
||||
"bufio" |
||||
"encoding/json" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"log" |
||||
"math/rand" |
||||
"os" |
||||
"path/filepath" |
||||
|
||||
"github.com/curusarn/resh/pkg/records" |
||||
) |
||||
|
||||
type deviceRecords struct { |
||||
Name string |
||||
Records []records.EnrichedRecord |
||||
} |
||||
|
||||
type userRecords struct { |
||||
Name string |
||||
Devices []deviceRecords |
||||
} |
||||
|
||||
// HistLoad loads history
|
||||
type HistLoad struct { |
||||
UsersRecords []userRecords |
||||
skipFailedCmds bool |
||||
sanitizedInput bool |
||||
debugRecords float64 |
||||
} |
||||
|
||||
func (e *HistLoad) preprocessDeviceRecords(device deviceRecords) deviceRecords { |
||||
sessionIDs := map[string]uint64{} |
||||
var nextID uint64 |
||||
nextID = 1 // start with 1 because 0 won't get saved to json
|
||||
for k, record := range device.Records { |
||||
id, found := sessionIDs[record.SessionID] |
||||
if found == false { |
||||
id = nextID |
||||
sessionIDs[record.SessionID] = id |
||||
nextID++ |
||||
} |
||||
device.Records[k].SeqSessionID = id |
||||
// assert
|
||||
if record.Sanitized != e.sanitizedInput { |
||||
if e.sanitizedInput { |
||||
log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") |
||||
} |
||||
log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") |
||||
} |
||||
device.Records[k].SeqSessionID = id |
||||
if e.debugRecords > 0 && rand.Float64() < e.debugRecords { |
||||
device.Records[k].DebugThisRecord = true |
||||
} |
||||
} |
||||
// sort.SliceStable(device.Records, func(x, y int) bool {
|
||||
// if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
|
||||
// return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal
|
||||
// }
|
||||
// return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID
|
||||
// })
|
||||
|
||||
// iterate from back and mark last record of each session
|
||||
sessionIDSet := map[string]bool{} |
||||
for i := len(device.Records) - 1; i >= 0; i-- { |
||||
var record *records.EnrichedRecord |
||||
record = &device.Records[i] |
||||
if sessionIDSet[record.SessionID] { |
||||
continue |
||||
} |
||||
sessionIDSet[record.SessionID] = true |
||||
record.LastRecordOfSession = true |
||||
} |
||||
return device |
||||
} |
||||
|
||||
// enrich records and add sequential session ID
|
||||
func (e *HistLoad) preprocessRecords() { |
||||
for i := range e.UsersRecords { |
||||
for j := range e.UsersRecords[i].Devices { |
||||
e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (e *HistLoad) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords { |
||||
var records []userRecords |
||||
info, err := os.Stat(dataRootPath) |
||||
if err != nil { |
||||
log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")") |
||||
} |
||||
if info.IsDir() == false { |
||||
log.Fatal("Error:", dataRootPath, "is not a directory - exiting!") |
||||
} |
||||
users, err := ioutil.ReadDir(dataRootPath) |
||||
if err != nil { |
||||
log.Fatal("Could not read directory:", dataRootPath) |
||||
} |
||||
fmt.Println("Listing users in <", dataRootPath, ">...") |
||||
for _, user := range users { |
||||
userRecords := userRecords{Name: user.Name()} |
||||
userFullPath := filepath.Join(dataRootPath, user.Name()) |
||||
if user.IsDir() == false { |
||||
log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.") |
||||
continue |
||||
} |
||||
fmt.Println() |
||||
fmt.Printf("*- %s\n", user.Name()) |
||||
devices, err := ioutil.ReadDir(userFullPath) |
||||
if err != nil { |
||||
log.Fatal("Could not read directory:", userFullPath) |
||||
} |
||||
for _, device := range devices { |
||||
deviceRecords := deviceRecords{Name: device.Name()} |
||||
deviceFullPath := filepath.Join(userFullPath, device.Name()) |
||||
if device.IsDir() == false { |
||||
log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.") |
||||
continue |
||||
} |
||||
fmt.Printf(" \\- %s\n", device.Name()) |
||||
files, err := ioutil.ReadDir(deviceFullPath) |
||||
if err != nil { |
||||
log.Fatal("Could not read directory:", deviceFullPath) |
||||
} |
||||
for _, file := range files { |
||||
fileFullPath := filepath.Join(deviceFullPath, file.Name()) |
||||
if file.Name() == fname { |
||||
fmt.Printf(" \\- %s - loading ...", file.Name()) |
||||
// load the data
|
||||
deviceRecords.Records = e.loadHistoryRecords(fileFullPath) |
||||
fmt.Println(" OK ✓") |
||||
} else { |
||||
fmt.Printf(" \\- %s - skipped\n", file.Name()) |
||||
} |
||||
} |
||||
userRecords.Devices = append(userRecords.Devices, deviceRecords) |
||||
} |
||||
records = append(records, userRecords) |
||||
} |
||||
return records |
||||
} |
||||
|
||||
func (e *HistLoad) loadHistoryRecords(fname string) []records.EnrichedRecord { |
||||
file, err := os.Open(fname) |
||||
if err != nil { |
||||
log.Fatal("Open() resh history file error:", err) |
||||
} |
||||
defer file.Close() |
||||
|
||||
var recs []records.EnrichedRecord |
||||
scanner := bufio.NewScanner(file) |
||||
for scanner.Scan() { |
||||
record := records.Record{} |
||||
fallbackRecord := records.FallbackRecord{} |
||||
line := scanner.Text() |
||||
err = json.Unmarshal([]byte(line), &record) |
||||
if err != nil { |
||||
err = json.Unmarshal([]byte(line), &fallbackRecord) |
||||
if err != nil { |
||||
log.Println("Line:", line) |
||||
log.Fatal("Decoding error:", err) |
||||
} |
||||
record = records.ConvertRecord(&fallbackRecord) |
||||
} |
||||
if e.sanitizedInput == false { |
||||
if record.CmdLength != 0 { |
||||
log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?") |
||||
} |
||||
record.CmdLength = len(record.CmdLine) |
||||
} |
||||
if record.CmdLength == 0 { |
||||
log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") |
||||
} |
||||
recs = append(recs, record.Enrich()) |
||||
} |
||||
return recs |
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
package strat |
||||
|
||||
import "github.com/curusarn/resh/pkg/records" |
||||
|
||||
type Dummy struct { |
||||
history []string |
||||
} |
||||
|
||||
func (s *Dummy) GetTitleAndDescription() (string, string) { |
||||
return "dummy", "Return empty candidate list" |
||||
} |
||||
|
||||
func (s *Dummy) GetCandidates() []string { |
||||
return nil |
||||
} |
||||
|
||||
func (s *Dummy) AddHistoryRecord(record *records.EnrichedRecord) error { |
||||
s.history = append(s.history, record.CmdLine) |
||||
return nil |
||||
} |
||||
|
||||
func (s *Dummy) ResetHistory() error { |
||||
return nil |
||||
} |
||||
@ -0,0 +1,44 @@ |
||||
package strat |
||||
|
||||
import ( |
||||
"github.com/curusarn/resh/pkg/records" |
||||
) |
||||
|
||||
type ISimpleStrategy interface { |
||||
GetTitleAndDescription() (string, string) |
||||
GetCandidates() []string |
||||
AddHistoryRecord(record *records.EnrichedRecord) error |
||||
ResetHistory() error |
||||
} |
||||
|
||||
type IStrategy interface { |
||||
GetTitleAndDescription() (string, string) |
||||
GetCandidates(r records.EnrichedRecord) []string |
||||
AddHistoryRecord(record *records.EnrichedRecord) error |
||||
ResetHistory() error |
||||
} |
||||
|
||||
type simpleStrategyWrapper struct { |
||||
strategy ISimpleStrategy |
||||
} |
||||
|
||||
// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy
|
||||
func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper { |
||||
return &simpleStrategyWrapper{strategy: strategy} |
||||
} |
||||
|
||||
func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) { |
||||
return s.strategy.GetTitleAndDescription() |
||||
} |
||||
|
||||
func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string { |
||||
return s.strategy.GetCandidates() |
||||
} |
||||
|
||||
func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error { |
||||
return s.strategy.AddHistoryRecord(r) |
||||
} |
||||
|
||||
func (s *simpleStrategyWrapper) ResetHistory() error { |
||||
return s.strategy.ResetHistory() |
||||
} |
||||
Loading…
Reference in new issue