diff --git a/.gitignore b/.gitignore index 602e54d..36f971e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ -resh-collect -resh-daemon -resh-sanitize-history -resh-evaluate +bin/* diff --git a/Makefile b/Makefile index 687ad36..353f9c1 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ SHELL=/bin/bash -VERSION=$(shell cat version) +VERSION=$(shell cat VERSION) REVISION=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_revision") GOFLAGS=-ldflags "-X main.Version=${VERSION} -X main.Revision=${REVISION}" autoinstall: - ./install_helper.sh + scripts/install_helper.sh sanitize: # @@ -23,8 +23,8 @@ sanitize: # # # Running history sanitization ... - resh-sanitize-history -trim-hashes 0 --output ~/resh_history_sanitized.json - resh-sanitize-history -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json + resh-sanitize -trim-hashes 0 --output ~/resh_history_sanitized.json + resh-sanitize -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json # # # SUCCESS - ALL DONE! @@ -41,8 +41,17 @@ sanitize: # # +build: test_go submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize -build: submodules resh-collect resh-daemon resh-sanitize-history resh-evaluate +test_go: + # Running tests + @for dir in {cmd,pkg}/* ; do \ + echo $$dir ; \ + go test $$dir/*.go ; \ + done + +test: + scripts/test.sh rebuild: make clean @@ -51,15 +60,15 @@ rebuild: clean: rm resh-* -install: build submodules/bash-preexec/bash-preexec.sh shellrc.sh config.toml uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config +install: build submodules/bash-preexec/bash-preexec.sh scripts/shellrc.sh conf/config.toml scripts/uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config # Copying files to resh directory ... cp -f submodules/bash-preexec/bash-preexec.sh ~/.bash-preexec.sh - cp -f config.toml ~/.config/resh.toml - cp -f shellrc.sh ~/.resh/shellrc - cp -f uuid.sh ~/.resh/bin/resh-uuid - cp -f resh-* ~/.resh/bin/ - cp -f evaluate/resh-evaluate-plot.py ~/.resh/bin/ - cp -fr sanitizer_data ~/.resh/ + cp -f conf/config.toml ~/.config/resh.toml + cp -f scripts/shellrc.sh ~/.resh/shellrc + cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid + cp -f bin/* ~/.resh/bin/ + cp -f scripts/resh-evaluate-plot.py ~/.resh/bin/ + cp -fr data/sanitizer ~/.resh/ # backward compatibility: We have a new location for resh history file [ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json # Adding resh shellrc to .bashrc ... @@ -107,17 +116,8 @@ uninstall: # Uninstalling ... -rm -rf ~/.resh/ -resh-daemon: daemon/resh-daemon.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< - -resh-collect: collect/resh-collect.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< - -resh-sanitize-history: sanitize-history/resh-sanitize-history.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< - -resh-evaluate: evaluate/resh-evaluate.go evaluate/strategy-*.go common/resh-common.go version - go build ${GOFLAGS} -o $@ $< evaluate/strategy-*.go +bin/resh-%: cmd/%/main.go pkg/*/*.go VERSION + go build ${GOFLAGS} -o $@ cmd/$*/*.go $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config: # Creating dirs ... @@ -129,7 +129,6 @@ $(HOME)/.resh/resh-uuid: .PHONY: submodules build install rebuild uninstall clean autoinstall - submodules: | submodules/bash-preexec/bash-preexec.sh @# sets submodule.recurse to true if unset @# sets status.submoduleSummary to true if unset diff --git a/README.md b/README.md index 74f86d3..dc0f620 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ If you are not happy with it you can uninstall it with a single command (`rm -rf The ultimate point of my thesis is to provide a context-based replacement/enhancement for bash and zsh shell history. The idea is to: + - Save each command with metadata (device, directory, git, time, terminal session pid, ... see example below) - Recommend history based on saved metadata - e.g. it will be easier to get to commands specific to the project you are currently working on (based on directory, git repository url, ...) @@ -45,9 +46,11 @@ If you install RESH, please give me some contact info using this form: https://f ## Installation ### Simplest -Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/rawinstall.sh)"` from anywhere. + +Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/scripts/rawinstall.sh)"` from anywhere. ### Simple + 1. Run `git clone https://github.com/curusarn/resh.git && cd resh` 2. Run `make autoinstall` for assisted build & instalation. - OR Run `make install` if you know how to build Golang projects. @@ -59,6 +62,7 @@ If you install RESH, please give me some contact info using this form: https://f Works in `bash` and `zsh`. Tested on: + - Arch - MacOS - Ubuntu (18.04) diff --git a/version b/VERSION similarity index 100% rename from version rename to VERSION diff --git a/collect/resh-collect.go b/cmd/collect/main.go similarity index 77% rename from collect/resh-collect.go rename to cmd/collect/main.go index b96cc72..75cc61d 100644 --- a/collect/resh-collect.go +++ b/cmd/collect/main.go @@ -11,7 +11,8 @@ import ( "os" "github.com/BurntSushi/toml" - common "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/pkg/records" // "os/exec" "os/user" @@ -30,11 +31,11 @@ func main() { usr, _ := user.Current() dir := usr.HomeDir configPath := filepath.Join(dir, "/.config/resh.toml") - reshUuidPath := filepath.Join(dir, "/.resh/resh-uuid") + reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid") - machineIdPath := "/etc/machine-id" + machineIDPath := "/etc/machine-id" - var config common.Config + var config cfg.Config if _, err := toml.DecodeFile(configPath, &config); err != nil { log.Fatal("Error reading config:", err) } @@ -48,7 +49,7 @@ func main() { exitCode := flag.Int("exitCode", -1, "exit code") shell := flag.String("shell", "", "actual shell") uname := flag.String("uname", "", "uname") - sessionId := flag.String("sessionId", "", "resh generated session id") + sessionID := flag.String("sessionId", "", "resh generated session id") // posix variables cols := flag.String("cols", "-1", "$COLUMNS") @@ -82,10 +83,10 @@ func main() { timezoneBefore := flag.String("timezoneBefore", "", "") timezoneAfter := flag.String("timezoneAfter", "", "") - osReleaseId := flag.String("osReleaseId", "", "/etc/os-release ID") - osReleaseVersionId := flag.String("osReleaseVersionId", "", + osReleaseID := flag.String("osReleaseId", "", "/etc/os-release ID") + osReleaseVersionID := flag.String("osReleaseVersionId", "", "/etc/os-release ID") - osReleaseIdLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") + osReleaseIDLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID") osReleaseName := flag.String("osReleaseName", "", "/etc/os-release ID") osReleasePrettyName := flag.String("osReleasePrettyName", "", "/etc/os-release ID") @@ -161,8 +162,8 @@ func main() { *gitRemote = "" } - if *osReleaseId == "" { - *osReleaseId = "linux" + if *osReleaseID == "" { + *osReleaseID = "linux" } if *osReleaseName == "" { *osReleaseName = "Linux" @@ -171,78 +172,80 @@ func main() { *osReleasePrettyName = "Linux" } - rec := common.Record{ - // core - CmdLine: *cmdLine, - ExitCode: *exitCode, - Shell: *shell, - Uname: *uname, - SessionId: *sessionId, - + rec := records.Record{ // posix Cols: *cols, Lines: *lines, - - Home: *home, - Lang: *lang, - LcAll: *lcAll, - Login: *login, - // Path: *path, - Pwd: *pwd, - PwdAfter: *pwdAfter, - ShellEnv: *shellEnv, - Term: *term, - - // non-posix - RealPwd: realPwd, - RealPwdAfter: realPwdAfter, - Pid: *pid, - SessionPid: *sessionPid, - Host: *host, - Hosttype: *hosttype, - Ostype: *ostype, - Machtype: *machtype, - Shlvl: *shlvl, - - // before after - TimezoneBefore: *timezoneBefore, - TimezoneAfter: *timezoneAfter, - - RealtimeBefore: realtimeBefore, - RealtimeAfter: realtimeAfter, - RealtimeBeforeLocal: realtimeBeforeLocal, - RealtimeAfterLocal: realtimeAfterLocal, - - RealtimeDuration: realtimeDuration, - RealtimeSinceSessionStart: realtimeSinceSessionStart, - RealtimeSinceBoot: realtimeSinceBoot, - - GitDir: gitDir, - GitRealDir: gitRealDir, - GitOriginRemote: *gitRemote, - MachineId: readFileContent(machineIdPath), - - OsReleaseId: *osReleaseId, - OsReleaseVersionId: *osReleaseVersionId, - OsReleaseIdLike: *osReleaseIdLike, - OsReleaseName: *osReleaseName, - OsReleasePrettyName: *osReleasePrettyName, - - ReshUuid: readFileContent(reshUuidPath), - ReshVersion: Version, - ReshRevision: Revision, + // core + BaseRecord: records.BaseRecord{ + CmdLine: *cmdLine, + ExitCode: *exitCode, + Shell: *shell, + Uname: *uname, + SessionID: *sessionID, + + // posix + Home: *home, + Lang: *lang, + LcAll: *lcAll, + Login: *login, + // Path: *path, + Pwd: *pwd, + PwdAfter: *pwdAfter, + ShellEnv: *shellEnv, + Term: *term, + + // non-posix + RealPwd: realPwd, + RealPwdAfter: realPwdAfter, + Pid: *pid, + SessionPid: *sessionPid, + Host: *host, + Hosttype: *hosttype, + Ostype: *ostype, + Machtype: *machtype, + Shlvl: *shlvl, + + // before after + TimezoneBefore: *timezoneBefore, + TimezoneAfter: *timezoneAfter, + + RealtimeBefore: realtimeBefore, + RealtimeAfter: realtimeAfter, + RealtimeBeforeLocal: realtimeBeforeLocal, + RealtimeAfterLocal: realtimeAfterLocal, + + RealtimeDuration: realtimeDuration, + RealtimeSinceSessionStart: realtimeSinceSessionStart, + RealtimeSinceBoot: realtimeSinceBoot, + + GitDir: gitDir, + GitRealDir: gitRealDir, + GitOriginRemote: *gitRemote, + MachineID: readFileContent(machineIDPath), + + OsReleaseID: *osReleaseID, + OsReleaseVersionID: *osReleaseVersionID, + OsReleaseIDLike: *osReleaseIDLike, + OsReleaseName: *osReleaseName, + OsReleasePrettyName: *osReleasePrettyName, + + ReshUUID: readFileContent(reshUUIDPath), + ReshVersion: Version, + ReshRevision: Revision, + }, } sendRecord(rec, strconv.Itoa(config.Port)) } -func sendRecord(r common.Record, port string) { - recJson, err := json.Marshal(r) +func sendRecord(r records.Record, port string) { + recJSON, err := json.Marshal(r) if err != nil { log.Fatal("send err 1", err) } req, err := http.NewRequest("POST", "http://localhost:"+port+"/record", - bytes.NewBuffer(recJson)) + bytes.NewBuffer(recJSON)) if err != nil { log.Fatal("send err 2", err) } @@ -279,14 +282,14 @@ func getGitDirs(cdup string, exitCode int, pwd string) (string, string) { func getTimezoneOffsetInSeconds(zone string) float64 { // date +%z -> "+0200" - hours_str := zone[:3] - mins_str := zone[3:] - hours, err := strconv.Atoi(hours_str) + hoursStr := zone[:3] + minsStr := zone[3:] + hours, err := strconv.Atoi(hoursStr) if err != nil { log.Println("err while parsing hours in timezone offset:", err) return -1 } - mins, err := strconv.Atoi(mins_str) + mins, err := strconv.Atoi(minsStr) if err != nil { log.Println("err while parsing mins in timezone offset:", err) return -1 diff --git a/daemon/resh-daemon.go b/cmd/daemon/main.go similarity index 97% rename from daemon/resh-daemon.go rename to cmd/daemon/main.go index c8bcb0e..9411350 100644 --- a/daemon/resh-daemon.go +++ b/cmd/daemon/main.go @@ -14,7 +14,8 @@ import ( "strings" "github.com/BurntSushi/toml" - common "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/cfg" + "github.com/curusarn/resh/pkg/records" ) // Version from git set during build @@ -43,7 +44,7 @@ func main() { log.SetOutput(f) log.SetPrefix(strconv.Itoa(os.Getpid()) + " | ") - var config common.Config + var config cfg.Config if _, err := toml.DecodeFile(configPath, &config); err != nil { log.Println("Error reading config", err) return @@ -88,7 +89,7 @@ type recordHandler struct { func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Write([]byte("OK\n")) - record := common.Record{} + record := records.Record{} jsn, err := ioutil.ReadAll(r.Body) if err != nil { diff --git a/cmd/evaluate/main.go b/cmd/evaluate/main.go new file mode 100644 index 0000000..7ae217f --- /dev/null +++ b/cmd/evaluate/main.go @@ -0,0 +1,152 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/user" + "path/filepath" + + "github.com/curusarn/resh/pkg/histanal" + "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/pkg/strat" +) + +// Version from git set during build +var Version string + +// Revision from git set during build +var Revision string + +func main() { + const maxCandidates = 50 + + usr, _ := user.Current() + dir := usr.HomeDir + historyPath := filepath.Join(dir, ".resh_history.json") + historyPathBatchMode := filepath.Join(dir, "resh_history.json") + sanitizedHistoryPath := filepath.Join(dir, "resh_history_sanitized.json") + // tmpPath := "/tmp/resh-evaluate-tmp.json" + + showVersion := flag.Bool("version", false, "Show version and exit") + showRevision := flag.Bool("revision", false, "Show git revision and exit") + input := flag.String("input", "", + "Input file (default: "+historyPath+"OR"+sanitizedHistoryPath+ + " depending on --sanitized-input option)") + // outputDir := flag.String("output", "/tmp/resh-evaluate", "Output directory") + sanitizedInput := flag.Bool("sanitized-input", false, + "Handle input as sanitized (also changes default value for input argument)") + plottingScript := flag.String("plotting-script", "resh-evaluate-plot.py", "Script to use for plotting") + inputDataRoot := flag.String("input-data-root", "", + "Input data root, enables batch mode, looks for files matching --input option") + slow := flag.Bool("slow", false, + "Enables strategies that takes a long time (e.g. markov chain strategies).") + skipFailedCmds := flag.Bool("skip-failed-cmds", false, + "Skips records with non-zero exit status.") + debugRecords := flag.Float64("debug", 0, "Debug records - percentage of records that should be debugged.") + + flag.Parse() + + // handle show{Version,Revision} options + if *showVersion == true { + fmt.Println(Version) + os.Exit(0) + } + if *showRevision == true { + fmt.Println(Revision) + os.Exit(0) + } + + // handle batch mode + batchMode := false + if *inputDataRoot != "" { + batchMode = true + } + // set default input + if *input == "" { + if *sanitizedInput { + *input = sanitizedHistoryPath + } else if batchMode { + *input = historyPathBatchMode + } else { + *input = historyPath + } + } + + var evaluator histanal.HistEval + if batchMode { + evaluator = histanal.NewHistEvalBatchMode(*input, *inputDataRoot, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) + } else { + evaluator = histanal.NewHistEval(*input, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput) + } + + var simpleStrategies []strat.ISimpleStrategy + var strategies []strat.IStrategy + + // dummy := strategyDummy{} + // simpleStrategies = append(simpleStrategies, &dummy) + + simpleStrategies = append(simpleStrategies, &strat.Recent{}) + + // frequent := strategyFrequent{} + // frequent.init() + // simpleStrategies = append(simpleStrategies, &frequent) + + // random := strategyRandom{candidatesSize: maxCandidates} + // random.init() + // simpleStrategies = append(simpleStrategies, &random) + + directory := strat.DirectorySensitive{} + directory.Init() + simpleStrategies = append(simpleStrategies, &directory) + + dynamicDistG := strat.DynamicRecordDistance{ + MaxDepth: 3000, + DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1, Git: 10}, + Label: "10*pwd,10*realpwd,session,time,10*git", + } + dynamicDistG.Init() + strategies = append(strategies, &dynamicDistG) + + distanceStaticBest := strat.RecordDistance{ + MaxDepth: 3000, + DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1}, + Label: "10*pwd,10*realpwd,session,time", + } + strategies = append(strategies, &distanceStaticBest) + + recentBash := strat.RecentBash{} + recentBash.Init() + strategies = append(strategies, &recentBash) + + if *slow { + + markovCmd := strat.MarkovChainCmd{Order: 1} + markovCmd.Init() + + markovCmd2 := strat.MarkovChainCmd{Order: 2} + markovCmd2.Init() + + markov := strat.MarkovChain{Order: 1} + markov.Init() + + markov2 := strat.MarkovChain{Order: 2} + markov2.Init() + + simpleStrategies = append(simpleStrategies, &markovCmd2, &markovCmd, &markov2, &markov) + } + + for _, strategy := range simpleStrategies { + strategies = append(strategies, strat.NewSimpleStrategyWrapper(strategy)) + } + + for _, strat := range strategies { + err := evaluator.Evaluate(strat) + if err != nil { + log.Println("Evaluator evaluate() error:", err) + } + } + + evaluator.CalculateStatsAndPlot(*plottingScript) +} diff --git a/sanitize-history/resh-sanitize-history.go b/cmd/sanitize/main.go similarity index 97% rename from sanitize-history/resh-sanitize-history.go rename to cmd/sanitize/main.go index 5ce0581..763f29a 100644 --- a/sanitize-history/resh-sanitize-history.go +++ b/cmd/sanitize/main.go @@ -19,7 +19,7 @@ import ( "strings" "unicode" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" giturls "github.com/whilp/git-urls" ) @@ -79,8 +79,8 @@ func main() { scanner := bufio.NewScanner(inputFile) for scanner.Scan() { - record := common.Record{} - fallbackRecord := common.FallbackRecord{} + record := records.Record{} + fallbackRecord := records.FallbackRecord{} line := scanner.Text() err = json.Unmarshal([]byte(line), &record) if err != nil { @@ -89,7 +89,7 @@ func main() { log.Println("Line:", line) log.Fatal("Decoding error:", err) } - record = common.ConvertRecord(&fallbackRecord) + record = records.ConvertRecord(&fallbackRecord) } err = sanitizer.sanitizeRecord(&record) if err != nil { @@ -139,7 +139,7 @@ func loadData(fname string) map[string]bool { return data } -func (s *sanitizer) sanitizeRecord(record *common.Record) error { +func (s *sanitizer) sanitizeRecord(record *records.Record) error { // hash directories of the paths record.Pwd = s.sanitizePath(record.Pwd) record.RealPwd = s.sanitizePath(record.RealPwd) @@ -153,7 +153,7 @@ func (s *sanitizer) sanitizeRecord(record *common.Record) error { // hash the most sensitive info, do not tokenize record.Host = s.hashToken(record.Host) record.Login = s.hashToken(record.Login) - record.MachineId = s.hashToken(record.MachineId) + record.MachineID = s.hashToken(record.MachineID) var err error // this changes git url a bit but I'm still happy with the result diff --git a/common/resh-common.go b/common/resh-common.go deleted file mode 100644 index 7e91094..0000000 --- a/common/resh-common.go +++ /dev/null @@ -1,247 +0,0 @@ -package common - -import ( - "log" - "strconv" - - "github.com/mattn/go-shellwords" -) - -// Record representing single executed command with its metadata -type Record struct { - // core - CmdLine string `json:"cmdLine"` - ExitCode int `json:"exitCode"` - Shell string `json:"shell"` - Uname string `json:"uname"` - SessionId string `json:"sessionId"` - - // posix - Cols string `json:"cols"` - Lines string `json:"lines"` - Home string `json:"home"` - Lang string `json:"lang"` - LcAll string `json:"lcAll"` - Login string `json:"login"` - //Path string `json:"path"` - Pwd string `json:"pwd"` - PwdAfter string `json:"pwdAfter"` - ShellEnv string `json:"shellEnv"` - Term string `json:"term"` - - // non-posix"` - RealPwd string `json:"realPwd"` - RealPwdAfter string `json:"realPwdAfter"` - Pid int `json:"pid"` - SessionPid int `json:"sessionPid"` - Host string `json:"host"` - Hosttype string `json:"hosttype"` - Ostype string `json:"ostype"` - Machtype string `json:"machtype"` - Shlvl int `json:"shlvl"` - - // before after - TimezoneBefore string `json:"timezoneBefore"` - TimezoneAfter string `json:"timezoneAfter"` - - RealtimeBefore float64 `json:"realtimeBefore"` - RealtimeAfter float64 `json:"realtimeAfter"` - RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` - RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` - - RealtimeDuration float64 `json:"realtimeDuration"` - RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` - RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` - //Logs []string `json: "logs"` - - GitDir string `json:"gitDir"` - GitRealDir string `json:"gitRealDir"` - GitOriginRemote string `json:"gitOriginRemote"` - MachineId string `json:"machineId"` - - OsReleaseId string `json:"osReleaseId"` - OsReleaseVersionId string `json:"osReleaseVersionId"` - OsReleaseIdLike string `json:"osReleaseIdLike"` - OsReleaseName string `json:"osReleaseName"` - OsReleasePrettyName string `json:"osReleasePrettyName"` - - ReshUuid string `json:"reshUuid"` - ReshVersion string `json:"reshVersion"` - ReshRevision string `json:"reshRevision"` - - // added by sanitizatizer - Sanitized bool `json:"sanitized"` - CmdLength int `json:"cmdLength,omitempty"` - - // enriching fields - added "later" - FirstWord string `json:"firstWord,omitempty"` - Invalid bool `json:"invalid,omitempty"` - SeqSessionID uint64 `json:"seqSessionID,omitempty"` -} - -// FallbackRecord when record is too old and can't be parsed into regular Record -type FallbackRecord struct { - // older version of the record where cols and lines are int - - // core - CmdLine string `json:"cmdLine"` - ExitCode int `json:"exitCode"` - Shell string `json:"shell"` - Uname string `json:"uname"` - SessionId string `json:"sessionId"` - - // posix - Cols int `json:"cols"` // notice the in type - Lines int `json:"lines"` // notice the in type - Home string `json:"home"` - Lang string `json:"lang"` - LcAll string `json:"lcAll"` - Login string `json:"login"` - //Path string `json:"path"` - Pwd string `json:"pwd"` - PwdAfter string `json:"pwdAfter"` - ShellEnv string `json:"shellEnv"` - Term string `json:"term"` - - // non-posix"` - RealPwd string `json:"realPwd"` - RealPwdAfter string `json:"realPwdAfter"` - Pid int `json:"pid"` - SessionPid int `json:"sessionPid"` - Host string `json:"host"` - Hosttype string `json:"hosttype"` - Ostype string `json:"ostype"` - Machtype string `json:"machtype"` - Shlvl int `json:"shlvl"` - - // before after - TimezoneBefore string `json:"timezoneBefore"` - TimezoneAfter string `json:"timezoneAfter"` - - RealtimeBefore float64 `json:"realtimeBefore"` - RealtimeAfter float64 `json:"realtimeAfter"` - RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` - RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` - - RealtimeDuration float64 `json:"realtimeDuration"` - RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` - RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` - //Logs []string `json: "logs"` - - GitDir string `json:"gitDir"` - GitRealDir string `json:"gitRealDir"` - GitOriginRemote string `json:"gitOriginRemote"` - MachineId string `json:"machineId"` - - OsReleaseId string `json:"osReleaseId"` - OsReleaseVersionId string `json:"osReleaseVersionId"` - OsReleaseIdLike string `json:"osReleaseIdLike"` - OsReleaseName string `json:"osReleaseName"` - OsReleasePrettyName string `json:"osReleasePrettyName"` - - ReshUuid string `json:"reshUuid"` - ReshVersion string `json:"reshVersion"` - ReshRevision string `json:"reshRevision"` -} - -// ConvertRecord from FallbackRecord to Record -func ConvertRecord(r *FallbackRecord) Record { - return Record{ - // core - CmdLine: r.CmdLine, - ExitCode: r.ExitCode, - Shell: r.Shell, - Uname: r.Uname, - SessionId: r.SessionId, - - // posix - // these two lines are the only reason we are doing this - Cols: strconv.Itoa(r.Cols), - Lines: strconv.Itoa(r.Lines), - - Home: r.Home, - Lang: r.Lang, - LcAll: r.LcAll, - Login: r.Login, - // Path: r.path, - Pwd: r.Pwd, - PwdAfter: r.PwdAfter, - ShellEnv: r.ShellEnv, - Term: r.Term, - - // non-posix - RealPwd: r.RealPwd, - RealPwdAfter: r.RealPwdAfter, - Pid: r.Pid, - SessionPid: r.SessionPid, - Host: r.Host, - Hosttype: r.Hosttype, - Ostype: r.Ostype, - Machtype: r.Machtype, - Shlvl: r.Shlvl, - - // before after - TimezoneBefore: r.TimezoneBefore, - TimezoneAfter: r.TimezoneAfter, - - RealtimeBefore: r.RealtimeBefore, - RealtimeAfter: r.RealtimeAfter, - RealtimeBeforeLocal: r.RealtimeBeforeLocal, - RealtimeAfterLocal: r.RealtimeAfterLocal, - - RealtimeDuration: r.RealtimeDuration, - RealtimeSinceSessionStart: r.RealtimeSinceSessionStart, - RealtimeSinceBoot: r.RealtimeSinceBoot, - - GitDir: r.GitDir, - GitRealDir: r.GitRealDir, - GitOriginRemote: r.GitOriginRemote, - MachineId: r.MachineId, - - OsReleaseId: r.OsReleaseId, - OsReleaseVersionId: r.OsReleaseVersionId, - OsReleaseIdLike: r.OsReleaseIdLike, - OsReleaseName: r.OsReleaseName, - OsReleasePrettyName: r.OsReleasePrettyName, - - ReshUuid: r.ReshUuid, - ReshVersion: r.ReshVersion, - ReshRevision: r.ReshRevision, - } -} - -// Enrich - adds additional fields to the record -func (r *Record) Enrich() { - // Get command/first word from commandline - r.FirstWord = GetCommandFromCommandLine(r.CmdLine) - err := r.Validate() - if err != nil { - log.Println("Invalid command:", r.CmdLine) - r.Invalid = true - } - r.Invalid = false - // TODO: Detect and mark simple commands r.Simple -} - -// Validate - returns error if the record is invalid -func (r *Record) Validate() error { - return nil -} - -// GetCommandFromCommandLine func -func GetCommandFromCommandLine(cmdLine string) string { - args, err := shellwords.Parse(cmdLine) - if err != nil { - log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") - return "" - } - if len(args) > 0 { - return args[0] - } - return "" -} - -// Config struct -type Config struct { - Port int -} diff --git a/config.toml b/conf/config.toml similarity index 100% rename from config.toml rename to conf/config.toml diff --git a/sanitizer_data/copyright_information.md b/data/sanitizer/copyright_information.md similarity index 100% rename from sanitizer_data/copyright_information.md rename to data/sanitizer/copyright_information.md diff --git a/sanitizer_data/whitelist.txt b/data/sanitizer/whitelist.txt similarity index 100% rename from sanitizer_data/whitelist.txt rename to data/sanitizer/whitelist.txt diff --git a/evaluate/resh-evaluate.go b/evaluate/resh-evaluate.go deleted file mode 100644 index bef0b24..0000000 --- a/evaluate/resh-evaluate.go +++ /dev/null @@ -1,340 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "os/exec" - "os/user" - "path/filepath" - "sort" - - "github.com/curusarn/resh/common" -) - -// Version from git set during build -var Version string - -// Revision from git set during build -var Revision string - -func main() { - usr, _ := user.Current() - dir := usr.HomeDir - historyPath := filepath.Join(dir, ".resh_history.json") - historyPathBatchMode := filepath.Join(dir, "resh_history.json") - sanitizedHistoryPath := filepath.Join(dir, "resh_history_sanitized.json") - // tmpPath := "/tmp/resh-evaluate-tmp.json" - - showVersion := flag.Bool("version", false, "Show version and exit") - showRevision := flag.Bool("revision", false, "Show git revision and exit") - input := flag.String("input", "", - "Input file (default: "+historyPath+"OR"+sanitizedHistoryPath+ - " depending on --sanitized-input option)") - // outputDir := flag.String("output", "/tmp/resh-evaluate", "Output directory") - sanitizedInput := flag.Bool("sanitized-input", false, - "Handle input as sanitized (also changes default value for input argument)") - plottingScript := flag.String("plotting-script", "resh-evaluate-plot.py", "Script to use for plotting") - inputDataRoot := flag.String("input-data-root", "", - "Input data root, enables batch mode, looks for files matching --input option") - - flag.Parse() - - // handle show{Version,Revision} options - if *showVersion == true { - fmt.Println(Version) - os.Exit(0) - } - if *showRevision == true { - fmt.Println(Revision) - os.Exit(0) - } - - // handle batch mode - batchMode := false - if *inputDataRoot != "" { - batchMode = true - } - // set default input - if *input == "" { - if *sanitizedInput { - *input = sanitizedHistoryPath - } else if batchMode { - *input = historyPathBatchMode - } else { - *input = historyPath - } - } - - evaluator := evaluator{sanitizedInput: *sanitizedInput, maxCandidates: 50, BatchMode: batchMode} - if batchMode { - err := evaluator.initBatchMode(*input, *inputDataRoot) - if err != nil { - log.Fatal("Evaluator initBatchMode() error:", err) - } - } else { - err := evaluator.init(*input) - if err != nil { - log.Fatal("Evaluator init() error:", err) - } - } - - var strategies []strategy - - // dummy := strategyDummy{} - // strategies = append(strategies, &dummy) - - recent := strategyRecent{} - frequent := strategyFrequent{} - frequent.init() - directory := strategyDirectorySensitive{} - directory.init() - - strategies = append(strategies, &recent, &frequent, &directory) - - for _, strat := range strategies { - err := evaluator.evaluate(strat) - if err != nil { - log.Println("Evaluator evaluate() error:", err) - } - } - - evaluator.calculateStatsAndPlot(*plottingScript) -} - -type strategy interface { - GetTitleAndDescription() (string, string) - GetCandidates() []string - AddHistoryRecord(record *common.Record) error - ResetHistory() error -} - -type matchJSON struct { - Match bool - Distance int - CharsRecalled int -} - -type strategyJSON struct { - Title string - Description string - Matches []matchJSON -} - -type deviceRecords struct { - Name string - Records []common.Record -} - -type userRecords struct { - Name string - Devices []deviceRecords -} - -type evaluator struct { - sanitizedInput bool - BatchMode bool - maxCandidates int - UsersRecords []userRecords - Strategies []strategyJSON -} - -func (e *evaluator) initBatchMode(input string, inputDataRoot string) error { - e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) - e.processRecords() - return nil -} - -func (e *evaluator) init(inputPath string) error { - records := e.loadHistoryRecords(inputPath) - device := deviceRecords{Records: records} - user := userRecords{} - user.Devices = append(user.Devices, device) - e.UsersRecords = append(e.UsersRecords, user) - e.processRecords() - return nil -} - -func (e *evaluator) calculateStatsAndPlot(scriptName string) { - evalJSON, err := json.Marshal(e) - if err != nil { - log.Fatal("json marshal error", err) - } - buffer := bytes.Buffer{} - buffer.Write(evalJSON) - // run python script to stat and plot/ - cmd := exec.Command(scriptName) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = &buffer - err = cmd.Run() - if err != nil { - log.Printf("Command finished with error: %v", err) - } -} - -// enrich records and add them to serializable structure -func (e *evaluator) processRecords() { - for i := range e.UsersRecords { - for j, device := range e.UsersRecords[i].Devices { - sessionIDs := map[string]uint64{} - var nextID uint64 - nextID = 0 - for k, record := range e.UsersRecords[i].Devices[j].Records { - id, found := sessionIDs[record.SessionId] - if found == false { - id = nextID - sessionIDs[record.SessionId] = id - nextID++ - } - record.SeqSessionID = id - // assert - if record.Sanitized != e.sanitizedInput { - if e.sanitizedInput { - log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") - } - log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") - } - - e.UsersRecords[i].Devices[j].Records[k].Enrich() - // device.Records = append(device.Records, record) - } - sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool { - if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { - return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal - } - return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID - }) - } - } -} - -func (e *evaluator) evaluate(strategy strategy) error { - title, description := strategy.GetTitleAndDescription() - strategyData := strategyJSON{Title: title, Description: description} - for _, record := range e.UsersRecords[0].Devices[0].Records { - candidates := strategy.GetCandidates() - - matchFound := false - for i, candidate := range candidates { - // make an option (--calculate-total) to turn this on/off ? - // if i >= e.maxCandidates { - // break - // } - if candidate == record.CmdLine { - match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} - strategyData.Matches = append(strategyData.Matches, match) - matchFound = true - break - } - } - if matchFound == false { - strategyData.Matches = append(strategyData.Matches, matchJSON{}) - } - err := strategy.AddHistoryRecord(&record) - if err != nil { - log.Println("Error while evauating", err) - return err - } - } - e.Strategies = append(e.Strategies, strategyData) - return nil -} - -func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords { - var records []userRecords - info, err := os.Stat(dataRootPath) - if err != nil { - log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")") - } - if info.IsDir() == false { - log.Fatal("Error:", dataRootPath, "is not a directory - exiting!") - } - users, err := ioutil.ReadDir(dataRootPath) - if err != nil { - log.Fatal("Could not read directory:", dataRootPath) - } - fmt.Println("Listing users in <", dataRootPath, ">...") - for _, user := range users { - userRecords := userRecords{Name: user.Name()} - userFullPath := filepath.Join(dataRootPath, user.Name()) - if user.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.") - continue - } - fmt.Println() - fmt.Printf("*- %s\n", user.Name()) - devices, err := ioutil.ReadDir(userFullPath) - if err != nil { - log.Fatal("Could not read directory:", userFullPath) - } - for _, device := range devices { - deviceRecords := deviceRecords{Name: device.Name()} - deviceFullPath := filepath.Join(userFullPath, device.Name()) - if device.IsDir() == false { - log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.") - continue - } - fmt.Printf(" \\- %s\n", device.Name()) - files, err := ioutil.ReadDir(deviceFullPath) - if err != nil { - log.Fatal("Could not read directory:", deviceFullPath) - } - for _, file := range files { - fileFullPath := filepath.Join(deviceFullPath, file.Name()) - if file.Name() == fname { - fmt.Printf(" \\- %s - loading ...", file.Name()) - // load the data - deviceRecords.Records = e.loadHistoryRecords(fileFullPath) - fmt.Println(" OK ✓") - } else { - fmt.Printf(" \\- %s - skipped\n", file.Name()) - } - } - userRecords.Devices = append(userRecords.Devices, deviceRecords) - } - records = append(records, userRecords) - } - return records -} - -func (e *evaluator) loadHistoryRecords(fname string) []common.Record { - file, err := os.Open(fname) - if err != nil { - log.Fatal("Open() resh history file error:", err) - } - defer file.Close() - - var records []common.Record - scanner := bufio.NewScanner(file) - for scanner.Scan() { - record := common.Record{} - fallbackRecord := common.FallbackRecord{} - line := scanner.Text() - err = json.Unmarshal([]byte(line), &record) - if err != nil { - err = json.Unmarshal([]byte(line), &fallbackRecord) - if err != nil { - log.Println("Line:", line) - log.Fatal("Decoding error:", err) - } - record = common.ConvertRecord(&fallbackRecord) - } - if e.sanitizedInput == false { - if record.CmdLength != 0 { - log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?") - } - record.CmdLength = len(record.CmdLine) - } - if record.CmdLength == 0 { - log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") - } - records = append(records, record) - } - return records -} diff --git a/evaluate/strategy-dummy.go b/evaluate/strategy-dummy.go deleted file mode 100644 index 28ed8ec..0000000 --- a/evaluate/strategy-dummy.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import "github.com/curusarn/resh/common" - -type strategyDummy struct { - history []string -} - -func (s *strategyDummy) GetTitleAndDescription() (string, string) { - return "dummy", "Return empty candidate list" -} - -func (s *strategyDummy) GetCandidates() []string { - return nil -} - -func (s *strategyDummy) AddHistoryRecord(record *common.Record) error { - s.history = append(s.history, record.CmdLine) - return nil -} - -func (s *strategyDummy) ResetHistory() error { - return nil -} diff --git a/evaluate/strategy-recent.go b/evaluate/strategy-recent.go deleted file mode 100644 index 7d24d23..0000000 --- a/evaluate/strategy-recent.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import "github.com/curusarn/resh/common" - -type strategyRecent struct { - history []string -} - -func (s *strategyRecent) GetTitleAndDescription() (string, string) { - return "recent", "Use recent commands" -} - -func (s *strategyRecent) GetCandidates() []string { - return s.history -} - -func (s *strategyRecent) AddHistoryRecord(record *common.Record) error { - // remove previous occurance of record - for i, cmd := range s.history { - if cmd == record.CmdLine { - s.history = append(s.history[:i], s.history[i+1:]...) - } - } - // append new record - s.history = append([]string{record.CmdLine}, s.history...) - return nil -} - -func (s *strategyRecent) ResetHistory() error { - s.history = nil - return nil -} diff --git a/go.mod b/go.mod index 9c901e1..b13b13d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,10 @@ go 1.12 require ( github.com/BurntSushi/toml v0.3.1 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 github.com/mattn/go-shellwords v1.0.6 + github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7 + github.com/schollz/progressbar v1.0.0 github.com/wcharczuk/go-chart v2.0.1+incompatible github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa golang.org/x/image v0.0.0-20190902063713-cb417be4ba39 // indirect diff --git a/go.sum b/go.sum index 92beac2..b5684fc 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 h1:1dSBUfGlorLAua2CRx0zFN7kQsTpE2DQSmr7rrTNgY8= +github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629/go.mod h1:mb5nS4uRANwOJSZj8rlCWAfAcGi72GGMIXx+xGOjA7M= github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7 h1:VsJjhYhufMGXICLwLYr8mFVMp8/A+YqmagMHnG/BA/4= +github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7/go.mod h1:zQmHoMvvVJb7cxyt1wGT77lqUaeOFXlogOppOr4uHVo= +github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= +github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= github.com/wcharczuk/go-chart v2.0.1+incompatible h1:0pz39ZAycJFF7ju/1mepnk26RLVLBCWz1STcD3doU0A= github.com/wcharczuk/go-chart v2.0.1+incompatible/go.mod h1:PF5tmL4EIx/7Wf+hEkpCqYi5He4u90sw+0+6FhrryuE= github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa h1:rW+Lu6281ed/4XGuVIa4/YebTRNvoUJlfJ44ktEVwZk= diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go new file mode 100644 index 0000000..8373306 --- /dev/null +++ b/pkg/cfg/cfg.go @@ -0,0 +1,6 @@ +package cfg + +// Config struct +type Config struct { + Port int +} diff --git a/pkg/histanal/histeval.go b/pkg/histanal/histeval.go new file mode 100644 index 0000000..4d19779 --- /dev/null +++ b/pkg/histanal/histeval.go @@ -0,0 +1,246 @@ +package histanal + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "math/rand" + "os" + "os/exec" + + "github.com/curusarn/resh/pkg/records" + "github.com/curusarn/resh/pkg/strat" + "github.com/jpillora/longestcommon" + + "github.com/schollz/progressbar" +) + +type matchJSON struct { + Match bool + Distance int + CharsRecalled int +} + +type multiMatchItemJSON struct { + Distance int + CharsRecalled int +} + +type multiMatchJSON struct { + Match bool + Entries []multiMatchItemJSON +} + +type strategyJSON struct { + Title string + Description string + Matches []matchJSON + PrefixMatches []multiMatchJSON +} + +// HistEval evaluates history +type HistEval struct { + HistLoad + BatchMode bool + maxCandidates int + Strategies []strategyJSON +} + +// NewHistEval constructs new HistEval +func NewHistEval(inputPath string, + maxCandidates int, skipFailedCmds bool, + debugRecords float64, sanitizedInput bool) HistEval { + + e := HistEval{ + HistLoad: HistLoad{ + skipFailedCmds: skipFailedCmds, + debugRecords: debugRecords, + sanitizedInput: sanitizedInput, + }, + maxCandidates: maxCandidates, + BatchMode: false, + } + records := e.loadHistoryRecords(inputPath) + device := deviceRecords{Records: records} + user := userRecords{} + user.Devices = append(user.Devices, device) + e.UsersRecords = append(e.UsersRecords, user) + e.preprocessRecords() + return e +} + +// NewHistEvalBatchMode constructs new HistEval in batch mode +func NewHistEvalBatchMode(input string, inputDataRoot string, + maxCandidates int, skipFailedCmds bool, + debugRecords float64, sanitizedInput bool) HistEval { + + e := HistEval{ + HistLoad: HistLoad{ + skipFailedCmds: skipFailedCmds, + debugRecords: debugRecords, + sanitizedInput: sanitizedInput, + }, + maxCandidates: maxCandidates, + BatchMode: false, + } + e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot) + e.preprocessRecords() + return e +} + +func (e *HistEval) preprocessDeviceRecords(device deviceRecords) deviceRecords { + sessionIDs := map[string]uint64{} + var nextID uint64 + nextID = 1 // start with 1 because 0 won't get saved to json + for k, record := range device.Records { + id, found := sessionIDs[record.SessionID] + if found == false { + id = nextID + sessionIDs[record.SessionID] = id + nextID++ + } + device.Records[k].SeqSessionID = id + // assert + if record.Sanitized != e.sanitizedInput { + if e.sanitizedInput { + log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") + } + log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") + } + device.Records[k].SeqSessionID = id + if e.debugRecords > 0 && rand.Float64() < e.debugRecords { + device.Records[k].DebugThisRecord = true + } + } + // sort.SliceStable(device.Records, func(x, y int) bool { + // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { + // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal + // } + // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID + // }) + + // iterate from back and mark last record of each session + sessionIDSet := map[string]bool{} + for i := len(device.Records) - 1; i >= 0; i-- { + var record *records.EnrichedRecord + record = &device.Records[i] + if sessionIDSet[record.SessionID] { + continue + } + sessionIDSet[record.SessionID] = true + record.LastRecordOfSession = true + } + return device +} + +// enrich records and add sequential session ID +func (e *HistEval) preprocessRecords() { + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) + } + } +} + +// Evaluate a given strategy +func (e *HistEval) Evaluate(strategy strat.IStrategy) error { + title, description := strategy.GetTitleAndDescription() + log.Println("Evaluating strategy:", title, "-", description) + strategyData := strategyJSON{Title: title, Description: description} + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records)) + var prevRecord records.EnrichedRecord + for _, record := range e.UsersRecords[i].Devices[j].Records { + if e.skipFailedCmds && record.ExitCode != 0 { + continue + } + candidates := strategy.GetCandidates(records.Stripped(record)) + if record.DebugThisRecord { + log.Println() + log.Println("===================================================") + log.Println("STRATEGY:", title, "-", description) + log.Println("===================================================") + log.Println("Previous record:") + if prevRecord.RealtimeBefore == 0 { + log.Println("== NIL") + } else { + rec, _ := prevRecord.ToString() + log.Println(rec) + } + log.Println("---------------------------------------------------") + log.Println("Recommendations for:") + rec, _ := record.ToString() + log.Println(rec) + log.Println("---------------------------------------------------") + for i, candidate := range candidates { + if i > 10 { + break + } + log.Println(string(candidate)) + } + log.Println("===================================================") + } + + matchFound := false + longestPrefixMatchLength := 0 + multiMatch := multiMatchJSON{} + for i, candidate := range candidates { + // make an option (--calculate-total) to turn this on/off ? + // if i >= e.maxCandidates { + // break + // } + commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine})) + if commonPrefixLength > longestPrefixMatchLength { + longestPrefixMatchLength = commonPrefixLength + prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength} + multiMatch.Match = true + multiMatch.Entries = append(multiMatch.Entries, prefixMatch) + } + if candidate == record.CmdLine { + match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength} + matchFound = true + strategyData.Matches = append(strategyData.Matches, match) + strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) + break + } + } + if matchFound == false { + strategyData.Matches = append(strategyData.Matches, matchJSON{}) + strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch) + } + err := strategy.AddHistoryRecord(&record) + if err != nil { + log.Println("Error while evauating", err) + return err + } + bar.Add(1) + prevRecord = record + } + strategy.ResetHistory() + fmt.Println() + } + } + e.Strategies = append(e.Strategies, strategyData) + return nil +} + +// CalculateStatsAndPlot results +func (e *HistEval) CalculateStatsAndPlot(scriptName string) { + evalJSON, err := json.Marshal(e) + if err != nil { + log.Fatal("json marshal error", err) + } + buffer := bytes.Buffer{} + buffer.Write(evalJSON) + // run python script to stat and plot/ + cmd := exec.Command(scriptName) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = &buffer + err = cmd.Run() + if err != nil { + log.Printf("Command finished with error: %v", err) + } +} diff --git a/pkg/histanal/histload.go b/pkg/histanal/histload.go new file mode 100644 index 0000000..313c7ff --- /dev/null +++ b/pkg/histanal/histload.go @@ -0,0 +1,179 @@ +package histanal + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "math/rand" + "os" + "path/filepath" + + "github.com/curusarn/resh/pkg/records" +) + +type deviceRecords struct { + Name string + Records []records.EnrichedRecord +} + +type userRecords struct { + Name string + Devices []deviceRecords +} + +// HistLoad loads history +type HistLoad struct { + UsersRecords []userRecords + skipFailedCmds bool + sanitizedInput bool + debugRecords float64 +} + +func (e *HistLoad) preprocessDeviceRecords(device deviceRecords) deviceRecords { + sessionIDs := map[string]uint64{} + var nextID uint64 + nextID = 1 // start with 1 because 0 won't get saved to json + for k, record := range device.Records { + id, found := sessionIDs[record.SessionID] + if found == false { + id = nextID + sessionIDs[record.SessionID] = id + nextID++ + } + device.Records[k].SeqSessionID = id + // assert + if record.Sanitized != e.sanitizedInput { + if e.sanitizedInput { + log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized") + } + log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present") + } + device.Records[k].SeqSessionID = id + if e.debugRecords > 0 && rand.Float64() < e.debugRecords { + device.Records[k].DebugThisRecord = true + } + } + // sort.SliceStable(device.Records, func(x, y int) bool { + // if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID { + // return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal + // } + // return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID + // }) + + // iterate from back and mark last record of each session + sessionIDSet := map[string]bool{} + for i := len(device.Records) - 1; i >= 0; i-- { + var record *records.EnrichedRecord + record = &device.Records[i] + if sessionIDSet[record.SessionID] { + continue + } + sessionIDSet[record.SessionID] = true + record.LastRecordOfSession = true + } + return device +} + +// enrich records and add sequential session ID +func (e *HistLoad) preprocessRecords() { + for i := range e.UsersRecords { + for j := range e.UsersRecords[i].Devices { + e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j]) + } + } +} + +func (e *HistLoad) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords { + var records []userRecords + info, err := os.Stat(dataRootPath) + if err != nil { + log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")") + } + if info.IsDir() == false { + log.Fatal("Error:", dataRootPath, "is not a directory - exiting!") + } + users, err := ioutil.ReadDir(dataRootPath) + if err != nil { + log.Fatal("Could not read directory:", dataRootPath) + } + fmt.Println("Listing users in <", dataRootPath, ">...") + for _, user := range users { + userRecords := userRecords{Name: user.Name()} + userFullPath := filepath.Join(dataRootPath, user.Name()) + if user.IsDir() == false { + log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.") + continue + } + fmt.Println() + fmt.Printf("*- %s\n", user.Name()) + devices, err := ioutil.ReadDir(userFullPath) + if err != nil { + log.Fatal("Could not read directory:", userFullPath) + } + for _, device := range devices { + deviceRecords := deviceRecords{Name: device.Name()} + deviceFullPath := filepath.Join(userFullPath, device.Name()) + if device.IsDir() == false { + log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.") + continue + } + fmt.Printf(" \\- %s\n", device.Name()) + files, err := ioutil.ReadDir(deviceFullPath) + if err != nil { + log.Fatal("Could not read directory:", deviceFullPath) + } + for _, file := range files { + fileFullPath := filepath.Join(deviceFullPath, file.Name()) + if file.Name() == fname { + fmt.Printf(" \\- %s - loading ...", file.Name()) + // load the data + deviceRecords.Records = e.loadHistoryRecords(fileFullPath) + fmt.Println(" OK ✓") + } else { + fmt.Printf(" \\- %s - skipped\n", file.Name()) + } + } + userRecords.Devices = append(userRecords.Devices, deviceRecords) + } + records = append(records, userRecords) + } + return records +} + +func (e *HistLoad) loadHistoryRecords(fname string) []records.EnrichedRecord { + file, err := os.Open(fname) + if err != nil { + log.Fatal("Open() resh history file error:", err) + } + defer file.Close() + + var recs []records.EnrichedRecord + scanner := bufio.NewScanner(file) + for scanner.Scan() { + record := records.Record{} + fallbackRecord := records.FallbackRecord{} + line := scanner.Text() + err = json.Unmarshal([]byte(line), &record) + if err != nil { + err = json.Unmarshal([]byte(line), &fallbackRecord) + if err != nil { + log.Println("Line:", line) + log.Fatal("Decoding error:", err) + } + record = records.ConvertRecord(&fallbackRecord) + } + if e.sanitizedInput == false { + if record.CmdLength != 0 { + log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?") + } + record.CmdLength = len(record.CmdLine) + } + if record.CmdLength == 0 { + log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.") + } + recs = append(recs, records.Enriched(record)) + } + return recs +} diff --git a/pkg/records/records.go b/pkg/records/records.go new file mode 100644 index 0000000..f15c717 --- /dev/null +++ b/pkg/records/records.go @@ -0,0 +1,391 @@ +package records + +import ( + "encoding/json" + "errors" + "log" + "math" + "strconv" + "strings" + + "github.com/mattn/go-shellwords" +) + +// BaseRecord - common base for Record and FallbackRecord +type BaseRecord struct { + // core + CmdLine string `json:"cmdLine"` + ExitCode int `json:"exitCode"` + Shell string `json:"shell"` + Uname string `json:"uname"` + SessionID string `json:"sessionId"` + + // posix + Home string `json:"home"` + Lang string `json:"lang"` + LcAll string `json:"lcAll"` + Login string `json:"login"` + //Path string `json:"path"` + Pwd string `json:"pwd"` + PwdAfter string `json:"pwdAfter"` + ShellEnv string `json:"shellEnv"` + Term string `json:"term"` + + // non-posix"` + RealPwd string `json:"realPwd"` + RealPwdAfter string `json:"realPwdAfter"` + Pid int `json:"pid"` + SessionPid int `json:"sessionPid"` + Host string `json:"host"` + Hosttype string `json:"hosttype"` + Ostype string `json:"ostype"` + Machtype string `json:"machtype"` + Shlvl int `json:"shlvl"` + + // before after + TimezoneBefore string `json:"timezoneBefore"` + TimezoneAfter string `json:"timezoneAfter"` + + RealtimeBefore float64 `json:"realtimeBefore"` + RealtimeAfter float64 `json:"realtimeAfter"` + RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"` + RealtimeAfterLocal float64 `json:"realtimeAfterLocal"` + + RealtimeDuration float64 `json:"realtimeDuration"` + RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"` + RealtimeSinceBoot float64 `json:"realtimeSinceBoot"` + //Logs []string `json: "logs"` + + GitDir string `json:"gitDir"` + GitRealDir string `json:"gitRealDir"` + GitOriginRemote string `json:"gitOriginRemote"` + MachineID string `json:"machineId"` + + OsReleaseID string `json:"osReleaseId"` + OsReleaseVersionID string `json:"osReleaseVersionId"` + OsReleaseIDLike string `json:"osReleaseIdLike"` + OsReleaseName string `json:"osReleaseName"` + OsReleasePrettyName string `json:"osReleasePrettyName"` + + ReshUUID string `json:"reshUuid"` + ReshVersion string `json:"reshVersion"` + ReshRevision string `json:"reshRevision"` + + // added by sanitizatizer + Sanitized bool `json:"sanitized,omitempty"` + CmdLength int `json:"cmdLength,omitempty"` +} + +// Record representing single executed command with its metadata +type Record struct { + BaseRecord + + Cols string `json:"cols"` + Lines string `json:"lines"` +} + +// EnrichedRecord - record enriched with additional data +type EnrichedRecord struct { + Record + + // enriching fields - added "later" + Command string `json:"command"` + FirstWord string `json:"firstWord"` + Invalid bool `json:"invalid"` + SeqSessionID uint64 `json:"seqSessionId"` + LastRecordOfSession bool `json:"lastRecordOfSession"` + DebugThisRecord bool `json:"debugThisRecord"` + Errors []string `json:"errors"` + // SeqSessionID uint64 `json:"seqSessionId,omitempty"` +} + +// FallbackRecord when record is too old and can't be parsed into regular Record +type FallbackRecord struct { + BaseRecord + // older version of the record where cols and lines are int + + Cols int `json:"cols"` // notice the int type + Lines int `json:"lines"` // notice the int type +} + +// ConvertRecord from FallbackRecord to Record +func ConvertRecord(r *FallbackRecord) Record { + return Record{ + BaseRecord: r.BaseRecord, + // these two lines are the only reason we are doing this + Cols: strconv.Itoa(r.Cols), + Lines: strconv.Itoa(r.Lines), + } +} + +// ToString - returns record the json +func (r EnrichedRecord) ToString() (string, error) { + jsonRec, err := json.Marshal(r) + if err != nil { + return "marshalling error", err + } + return string(jsonRec), nil +} + +// Enriched - returnd enriched record +func Enriched(r Record) EnrichedRecord { + record := EnrichedRecord{Record: r} + // Get command/first word from commandline + var err error + record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine) + if err != nil { + record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error()) + rec, _ := record.ToString() + log.Println("Invalid command:", rec) + record.Invalid = true + return record + } + err = r.Validate() + if err != nil { + record.Errors = append(record.Errors, "Validate error:"+err.Error()) + rec, _ := record.ToString() + log.Println("Invalid command:", rec) + record.Invalid = true + } + return record + // TODO: Detect and mark simple commands r.Simple +} + +// Validate - returns error if the record is invalid +func (r *Record) Validate() error { + if r.CmdLine == "" { + return errors.New("There is no CmdLine") + } + if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 { + return errors.New("There is no Time") + } + if r.RealtimeBeforeLocal == 0 || r.RealtimeAfterLocal == 0 { + return errors.New("There is no Local Time") + } + if r.RealPwd == "" || r.RealPwdAfter == "" { + return errors.New("There is no Real Pwd") + } + if r.Pwd == "" || r.PwdAfter == "" { + return errors.New("There is no Pwd") + } + + // TimezoneBefore + // TimezoneAfter + + // RealtimeDuration + // RealtimeSinceSessionStart - TODO: add later + // RealtimeSinceBoot - TODO: add later + + // device extras + // Host + // Hosttype + // Ostype + // Machtype + // OsReleaseID + // OsReleaseVersionID + // OsReleaseIDLike + // OsReleaseName + // OsReleasePrettyName + + // session extras + // Term + // Shlvl + + // static info + // Lang + // LcAll + + // meta + // ReshUUID + // ReshVersion + // ReshRevision + + // added by sanitizatizer + // Sanitized + // CmdLength + return nil +} + +// SetCmdLine sets cmdLine and related members +func (r *EnrichedRecord) SetCmdLine(cmdLine string) { + r.CmdLine = cmdLine + r.CmdLength = len(cmdLine) + r.ExitCode = 0 + var err error + r.Command, r.FirstWord, err = GetCommandAndFirstWord(cmdLine) + if err != nil { + r.Errors = append(r.Errors, "GetCommandAndFirstWord error:"+err.Error()) + // log.Println("Invalid command:", r.CmdLine) + r.Invalid = true + } +} + +// Stripped returns record stripped of all info that is not available during prediction +func Stripped(r EnrichedRecord) EnrichedRecord { + // clear the cmd itself + r.SetCmdLine("") + // replace after info with before info + r.PwdAfter = r.Pwd + r.RealPwdAfter = r.RealPwd + r.TimezoneAfter = r.TimezoneBefore + r.RealtimeAfter = r.RealtimeBefore + r.RealtimeAfterLocal = r.RealtimeBeforeLocal + // clear some more stuff + r.RealtimeDuration = 0 + r.LastRecordOfSession = false + return r +} + +// GetCommandAndFirstWord func +func GetCommandAndFirstWord(cmdLine string) (string, string, error) { + args, err := shellwords.Parse(cmdLine) + if err != nil { + log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )") + return "", "", err + } + if len(args) == 0 { + return "", "", nil + } + i := 0 + for true { + // commands in shell sometimes look like this `variable=something command argument otherArgument --option` + // to get the command we skip over tokens that contain '=' + if strings.ContainsRune(args[i], '=') && len(args) > i+1 { + i++ + continue + } + return args[i], args[0], nil + } + log.Fatal("GetCommandAndFirstWord error: this should not happen!") + return "ERROR", "ERROR", errors.New("this should not happen - contact developer ;)") +} + +// DistParams is used to supply params to Enrichedrecords.DistanceTo() +type DistParams struct { + ExitCode float64 + MachineID float64 + SessionID float64 + Login float64 + Shell float64 + Pwd float64 + RealPwd float64 + Git float64 + Time float64 +} + +// DistanceTo another record +func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 { + var dist float64 + dist = 0 + + // lev distance or something? TODO later + // CmdLine + + // exit code + if r.ExitCode != r2.ExitCode { + if r.ExitCode == 0 || r2.ExitCode == 0 { + // one success + one error -> 1 + dist += 1 * p.ExitCode + } else { + // two different errors + dist += 0.5 * p.ExitCode + } + } + + // machine/device + if r.MachineID != r2.MachineID { + dist += 1 * p.MachineID + } + // Uname + + // session + if r.SessionID != r2.SessionID { + dist += 1 * p.SessionID + } + // Pid - add because of nested shells? + // SessionPid + + // user + if r.Login != r2.Login { + dist += 1 * p.Login + } + // Home + + // shell + if r.Shell != r2.Shell { + dist += 1 * p.Shell + } + // ShellEnv + + // pwd + if r.Pwd != r2.Pwd { + // TODO: compare using hierarchy + // TODO: make more important + dist += 1 * p.Pwd + } + if r.RealPwd != r2.RealPwd { + // TODO: -||- + dist += 1 * p.RealPwd + } + // PwdAfter + // RealPwdAfter + + // git + if r.GitDir != r2.GitDir { + dist += 1 * p.Git + } + if r.GitRealDir != r2.GitRealDir { + dist += 1 * p.Git + } + if r.GitOriginRemote != r2.GitOriginRemote { + dist += 1 * p.Git + } + + // time + // this can actually get negative for differences of less than one second which is fine + // distance grows by 1 with every order + distTime := math.Log10(math.Abs(r.RealtimeBefore-r2.RealtimeBefore)) * p.Time + if math.IsNaN(distTime) == false && math.IsInf(distTime, 0) == false { + dist += distTime + } + // RealtimeBeforeLocal + // RealtimeAfter + // RealtimeAfterLocal + + // TimezoneBefore + // TimezoneAfter + + // RealtimeDuration + // RealtimeSinceSessionStart - TODO: add later + // RealtimeSinceBoot - TODO: add later + + // device extras + // Host + // Hosttype + // Ostype + // Machtype + // OsReleaseID + // OsReleaseVersionID + // OsReleaseIDLike + // OsReleaseName + // OsReleasePrettyName + + // session extras + // Term + // Shlvl + + // static info + // Lang + // LcAll + + // meta + // ReshUUID + // ReshVersion + // ReshRevision + + // added by sanitizatizer + // Sanitized + // CmdLength + + return dist +} diff --git a/pkg/records/records_test.go b/pkg/records/records_test.go new file mode 100644 index 0000000..5ef3c55 --- /dev/null +++ b/pkg/records/records_test.go @@ -0,0 +1,152 @@ +package records + +import ( + "bufio" + "encoding/json" + "log" + "os" + "testing" +) + +func GetTestRecords() []Record { + file, err := os.Open("testdata/resh_history.json") + if err != nil { + log.Fatal("Open() resh history file error:", err) + } + defer file.Close() + + var recs []Record + scanner := bufio.NewScanner(file) + for scanner.Scan() { + record := Record{} + line := scanner.Text() + err = json.Unmarshal([]byte(line), &record) + if err != nil { + log.Println("Line:", line) + log.Fatal("Decoding error:", err) + } + recs = append(recs, record) + } + return recs +} + +func GetTestEnrichedRecords() []EnrichedRecord { + var recs []EnrichedRecord + for _, rec := range GetTestRecords() { + recs = append(recs, Enriched(rec)) + } + return recs +} + +func TestToString(t *testing.T) { + for _, rec := range GetTestEnrichedRecords() { + _, err := rec.ToString() + if err != nil { + t.Error("ToString() failed") + } + } +} + +func TestEnriched(t *testing.T) { + record := Record{BaseRecord: BaseRecord{CmdLine: "cmd arg1 arg2"}} + enriched := Enriched(record) + if enriched.FirstWord != "cmd" || enriched.Command != "cmd" { + t.Error("Enriched() returned reocord w/ wrong Command OR FirstWord") + } +} + +func TestValidate(t *testing.T) { + record := EnrichedRecord{} + if record.Validate() == nil { + t.Error("Validate() didn't return an error for invalid record") + } + record.CmdLine = "cmd arg" + record.FirstWord = "cmd" + record.Command = "cmd" + time := 1234.5678 + record.RealtimeBefore = time + record.RealtimeAfter = time + record.RealtimeBeforeLocal = time + record.RealtimeAfterLocal = time + pwd := "/pwd" + record.Pwd = pwd + record.PwdAfter = pwd + record.RealPwd = pwd + record.RealPwdAfter = pwd + if record.Validate() != nil { + t.Error("Validate() returned an error for a valid record") + } +} + +func TestSetCmdLine(t *testing.T) { + record := EnrichedRecord{} + cmdline := "cmd arg1 arg2" + record.SetCmdLine(cmdline) + if record.CmdLine != cmdline || record.Command != "cmd" || record.FirstWord != "cmd" { + t.Error() + } +} + +func TestStripped(t *testing.T) { + for _, rec := range GetTestEnrichedRecords() { + stripped := Stripped(rec) + + // there should be no cmdline + if stripped.CmdLine != "" || + stripped.FirstWord != "" || + stripped.Command != "" { + t.Error("Stripped() returned record w/ info about CmdLine, Command OR FirstWord") + } + // *after* fields should be overwritten by *before* fields + if stripped.PwdAfter != stripped.Pwd || + stripped.RealPwdAfter != stripped.RealPwd || + stripped.TimezoneAfter != stripped.TimezoneBefore || + stripped.RealtimeAfter != stripped.RealtimeBefore || + stripped.RealtimeAfterLocal != stripped.RealtimeBeforeLocal { + t.Error("Stripped() returned record w/ different *after* and *before* values - *after* fields should be overwritten by *before* fields") + } + // there should be no information about duration and session end + if stripped.RealtimeDuration != 0 || + stripped.LastRecordOfSession != false { + t.Error("Stripped() returned record with too much information") + } + } +} + +func TestGetCommandAndFirstWord(t *testing.T) { + cmd, stWord, err := GetCommandAndFirstWord("cmd arg1 arg2") + if err != nil || cmd != "cmd" || stWord != "cmd" { + t.Error("GetCommandAndFirstWord() returned wrong Command OR FirstWord") + } +} + +func TestDistanceTo(t *testing.T) { + paramsFull := DistParams{ + ExitCode: 1, + MachineID: 1, + SessionID: 1, + Login: 1, + Shell: 1, + Pwd: 1, + RealPwd: 1, + Git: 1, + Time: 1, + } + paramsZero := DistParams{} + var prevRec EnrichedRecord + for _, rec := range GetTestEnrichedRecords() { + dist := rec.DistanceTo(rec, paramsFull) + if dist != 0 { + t.Error("DistanceTo() itself should be always 0") + } + dist = rec.DistanceTo(prevRec, paramsFull) + if dist == 0 { + t.Error("DistanceTo() between two test records shouldn't be 0") + } + dist = rec.DistanceTo(prevRec, paramsZero) + if dist != 0 { + t.Error("DistanceTo() should be 0 when DistParams is all zeros") + } + prevRec = rec + } +} diff --git a/pkg/records/testdata/resh_history.json b/pkg/records/testdata/resh_history.json new file mode 100644 index 0000000..40f43ab --- /dev/null +++ b/pkg/records/testdata/resh_history.json @@ -0,0 +1,27 @@ +{"cmdLine":"ls","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"d5c0fe70-c80b-4715-87cb-f8d8d5b4c673","cols":"80","lines":"24","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14560,"sessionPid":14560,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1566762905.173595,"realtimeAfter":1566762905.1894295,"realtimeBeforeLocal":1566770105.173595,"realtimeAfterLocal":1566770105.1894295,"realtimeDuration":0.015834569931030273,"realtimeSinceSessionStart":1.7122540473937988,"realtimeSinceBoot":20766.542254047396,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"find . -name applications","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567420001.2531302,"realtimeAfter":1567420002.4311218,"realtimeBeforeLocal":1567427201.2531302,"realtimeAfterLocal":1567427202.4311218,"realtimeDuration":1.1779916286468506,"realtimeSinceSessionStart":957.4848053455353,"realtimeSinceBoot":2336.594805345535,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"desktop-file-validate curusarn.sync-clipboards.desktop ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/.local/share/applications","pwdAfter":"/home/simon/.local/share/applications","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/.local/share/applications","realPwdAfter":"/home/simon/.local/share/applications","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567421748.2965438,"realtimeAfter":1567421748.3068867,"realtimeBeforeLocal":1567428948.2965438,"realtimeAfterLocal":1567428948.3068867,"realtimeDuration":0.010342836380004883,"realtimeSinceSessionStart":2704.528218984604,"realtimeSinceBoot":4083.6382189846036,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"cat /tmp/extensions | grep '.'","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461416.6871984,"realtimeAfter":1567461416.7336714,"realtimeBeforeLocal":1567468616.6871984,"realtimeAfterLocal":1567468616.7336714,"realtimeDuration":0.046473026275634766,"realtimeSinceSessionStart":21.45597553253174,"realtimeSinceBoot":43752.03597553253,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"cd git/resh/","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461667.8806899,"realtimeAfter":1567461667.8949044,"realtimeBeforeLocal":1567468867.8806899,"realtimeAfterLocal":1567468867.8949044,"realtimeDuration":0.014214515686035156,"realtimeSinceSessionStart":272.64946699142456,"realtimeSinceBoot":44003.229466991426,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461707.6467602,"realtimeAfter":1567461707.7177293,"realtimeBeforeLocal":1567468907.6467602,"realtimeAfterLocal":1567468907.7177293,"realtimeDuration":0.0709691047668457,"realtimeSinceSessionStart":312.4155373573303,"realtimeSinceBoot":44042.99553735733,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"cat /tmp/extensions | grep '^\\.' | cut -f1 |tr '[:upper:]' '[:lower:]' ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461722.813049,"realtimeAfter":1567461722.8280325,"realtimeBeforeLocal":1567468922.813049,"realtimeAfterLocal":1567468922.8280325,"realtimeDuration":0.014983415603637695,"realtimeSinceSessionStart":327.581826210022,"realtimeSinceBoot":44058.161826210024,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"tig","exitCode":127,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461906.3896828,"realtimeAfter":1567461906.4084594,"realtimeBeforeLocal":1567469106.3896828,"realtimeAfterLocal":1567469106.4084594,"realtimeDuration":0.018776655197143555,"realtimeSinceSessionStart":511.1584599018097,"realtimeSinceBoot":44241.73845990181,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"} +{"cmdLine":"resh-sanitize-history | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a3318c80-3521-4b22-aa64-ea0f6c641410","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14601,"sessionPid":14601,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567547116.2430356,"realtimeAfter":1567547116.7547352,"realtimeBeforeLocal":1567554316.2430356,"realtimeAfterLocal":1567554316.7547352,"realtimeDuration":0.5116996765136719,"realtimeSinceSessionStart":15.841878414154053,"realtimeSinceBoot":30527.201878414155,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo pacman -S ansible","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609042.0166302,"realtimeAfter":1567609076.9726007,"realtimeBeforeLocal":1567616242.0166302,"realtimeAfterLocal":1567616276.9726007,"realtimeDuration":34.95597052574158,"realtimeSinceSessionStart":1617.0794131755829,"realtimeSinceBoot":6120.029413175583,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"vagrant up","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609090.7359188,"realtimeAfter":1567609098.3125577,"realtimeBeforeLocal":1567616290.7359188,"realtimeAfterLocal":1567616298.3125577,"realtimeDuration":7.57663893699646,"realtimeSinceSessionStart":1665.798701763153,"realtimeSinceBoot":6168.748701763153,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo modprobe vboxnetflt","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609143.2847652,"realtimeAfter":1567609143.3116078,"realtimeBeforeLocal":1567616343.2847652,"realtimeAfterLocal":1567616343.3116078,"realtimeDuration":0.026842594146728516,"realtimeSinceSessionStart":1718.3475482463837,"realtimeSinceBoot":6221.2975482463835,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"echo $RANDOM","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"8ddacadc-6e73-483c-b347-4e18df204466","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":31387,"sessionPid":31387,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567727039.6540458,"realtimeAfter":1567727039.6629689,"realtimeBeforeLocal":1567734239.6540458,"realtimeAfterLocal":1567734239.6629689,"realtimeDuration":0.008923053741455078,"realtimeSinceSessionStart":1470.7667458057404,"realtimeSinceBoot":18495.01674580574,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"make resh-evaluate ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567977478.9672194,"realtimeAfter":1567977479.5449634,"realtimeBeforeLocal":1567984678.9672194,"realtimeAfterLocal":1567984679.5449634,"realtimeDuration":0.5777440071105957,"realtimeSinceSessionStart":5738.577540636063,"realtimeSinceBoot":20980.42754063606,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"cat ~/.resh_history.json | grep \"./resh-eval\" | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"105","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567986105.3988302,"realtimeAfter":1567986105.4809113,"realtimeBeforeLocal":1567993305.3988302,"realtimeAfterLocal":1567993305.4809113,"realtimeDuration":0.08208107948303223,"realtimeSinceSessionStart":14365.00915145874,"realtimeSinceBoot":29606.85915145874,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git c \"add sanitized flag to record, add Enrich() to record\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063976.9103937,"realtimeAfter":1568063976.9326868,"realtimeBeforeLocal":1568071176.9103937,"realtimeAfterLocal":1568071176.9326868,"realtimeDuration":0.0222930908203125,"realtimeSinceSessionStart":92236.52071499825,"realtimeSinceBoot":107478.37071499825,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063978.2340608,"realtimeAfter":1568063978.252463,"realtimeBeforeLocal":1568071178.2340608,"realtimeAfterLocal":1568071178.252463,"realtimeDuration":0.0184023380279541,"realtimeSinceSessionStart":92237.84438204765,"realtimeSinceBoot":107479.69438204766,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git a evaluate/results.go ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063989.0446353,"realtimeAfter":1568063989.2452207,"realtimeBeforeLocal":1568071189.0446353,"realtimeAfterLocal":1568071189.2452207,"realtimeDuration":0.20058536529541016,"realtimeSinceSessionStart":92248.65495657921,"realtimeSinceBoot":107490.50495657921,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo pacman -S python-pip","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072068.3557143,"realtimeAfter":1568072070.7509863,"realtimeBeforeLocal":1568079268.3557143,"realtimeAfterLocal":1568079270.7509863,"realtimeDuration":2.3952720165252686,"realtimeSinceSessionStart":100327.96603560448,"realtimeSinceBoot":115569.81603560448,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"pip3 install matplotlib","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072088.5575967,"realtimeAfter":1568072094.372314,"realtimeBeforeLocal":1568079288.5575967,"realtimeAfterLocal":1568079294.372314,"realtimeDuration":5.8147172927856445,"realtimeSinceSessionStart":100348.16791796684,"realtimeSinceBoot":115590.01791796685,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"sudo pip3 install matplotlib","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072106.138616,"realtimeAfter":1568072115.1124601,"realtimeBeforeLocal":1568079306.138616,"realtimeAfterLocal":1568079315.1124601,"realtimeDuration":8.973844051361084,"realtimeSinceSessionStart":100365.7489373684,"realtimeSinceBoot":115607.5989373684,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"./resh-evaluate --plotting-script evaluate/resh-evaluate-plot.py --input ~/git/resh_private/history_data/simon/dell/resh_history.json ","exitCode":130,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568076266.9364285,"realtimeAfter":1568076288.1131275,"realtimeBeforeLocal":1568083466.9364285,"realtimeAfterLocal":1568083488.1131275,"realtimeDuration":21.176698923110962,"realtimeSinceSessionStart":104526.54674983025,"realtimeSinceBoot":119768.39674983025,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0} +{"cmdLine":"git c \"Add a bunch of useless comments to make linter happy\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"04050353-a97d-4435-9248-f47dd08b2f2a","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":14702,"sessionPid":14702,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569456045.8763022,"realtimeAfter":1569456045.9030173,"realtimeBeforeLocal":1569463245.8763022,"realtimeAfterLocal":1569463245.9030173,"realtimeDuration":0.02671504020690918,"realtimeSinceSessionStart":2289.789242744446,"realtimeSinceBoot":143217.91924274445,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"fuck","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a4aadf03-610d-4731-ba94-5b7ce21e7bb9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":3413,"sessionPid":3413,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569687682.4250975,"realtimeAfter":1569687682.5877323,"realtimeBeforeLocal":1569694882.4250975,"realtimeAfterLocal":1569694882.5877323,"realtimeDuration":0.16263484954833984,"realtimeSinceSessionStart":264603.49496507645,"realtimeSinceBoot":374854.48496507644,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"code .","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709366.523767,"realtimeAfter":1569709367.516908,"realtimeBeforeLocal":1569716566.523767,"realtimeAfterLocal":1569716567.516908,"realtimeDuration":0.9931409358978271,"realtimeSinceSessionStart":23846.908839941025,"realtimeSinceBoot":396539.888839941,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"make test","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709371.89966,"realtimeAfter":1569709377.430194,"realtimeBeforeLocal":1569716571.89966,"realtimeAfterLocal":1569716577.430194,"realtimeDuration":5.530533790588379,"realtimeSinceSessionStart":23852.284733057022,"realtimeSinceBoot":396545.264733057,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} +{"cmdLine":"mkdir ~/git/resh/testdata","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"71529b60-2e7b-4d5b-8dc1-6d0740b58e9e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":21224,"sessionPid":21224,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709838.4642656,"realtimeAfter":1569709838.4718792,"realtimeBeforeLocal":1569717038.4642656,"realtimeAfterLocal":1569717038.4718792,"realtimeDuration":0.007613658905029297,"realtimeSinceSessionStart":9.437154054641724,"realtimeSinceBoot":397011.02715405467,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false} diff --git a/evaluate/strategy-directory-sensitive.go b/pkg/strat/directory-sensitive.go similarity index 51% rename from evaluate/strategy-directory-sensitive.go rename to pkg/strat/directory-sensitive.go index 0c00bc4..89d030e 100644 --- a/evaluate/strategy-directory-sensitive.go +++ b/pkg/strat/directory-sensitive.go @@ -1,27 +1,30 @@ -package main +package strat -import ( - "github.com/curusarn/resh/common" -) +import "github.com/curusarn/resh/pkg/records" -type strategyDirectorySensitive struct { +// DirectorySensitive prediction/recommendation strategy +type DirectorySensitive struct { history map[string][]string lastPwd string } -func (s *strategyDirectorySensitive) init() { +// Init see name +func (s *DirectorySensitive) Init() { s.history = map[string][]string{} } -func (s *strategyDirectorySensitive) GetTitleAndDescription() (string, string) { +// GetTitleAndDescription see name +func (s *DirectorySensitive) GetTitleAndDescription() (string, string) { return "directory sensitive (recent)", "Use recent commands executed is the same directory" } -func (s *strategyDirectorySensitive) GetCandidates() []string { +// GetCandidates see name +func (s *DirectorySensitive) GetCandidates() []string { return s.history[s.lastPwd] } -func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) error { +// AddHistoryRecord see name +func (s *DirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error { // work on history for PWD pwd := record.Pwd // remove previous occurance of record @@ -36,7 +39,9 @@ func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) err return nil } -func (s *strategyDirectorySensitive) ResetHistory() error { +// ResetHistory see name +func (s *DirectorySensitive) ResetHistory() error { + s.Init() s.history = map[string][]string{} return nil } diff --git a/pkg/strat/dummy.go b/pkg/strat/dummy.go new file mode 100644 index 0000000..fc813f2 --- /dev/null +++ b/pkg/strat/dummy.go @@ -0,0 +1,29 @@ +package strat + +import "github.com/curusarn/resh/pkg/records" + +// Dummy prediction/recommendation strategy +type Dummy struct { + history []string +} + +// GetTitleAndDescription see name +func (s *Dummy) GetTitleAndDescription() (string, string) { + return "dummy", "Return empty candidate list" +} + +// GetCandidates see name +func (s *Dummy) GetCandidates() []string { + return nil +} + +// AddHistoryRecord see name +func (s *Dummy) AddHistoryRecord(record *records.EnrichedRecord) error { + s.history = append(s.history, record.CmdLine) + return nil +} + +// ResetHistory see name +func (s *Dummy) ResetHistory() error { + return nil +} diff --git a/pkg/strat/dynamic-record-distance.go b/pkg/strat/dynamic-record-distance.go new file mode 100644 index 0000000..1f779c2 --- /dev/null +++ b/pkg/strat/dynamic-record-distance.go @@ -0,0 +1,91 @@ +package strat + +import ( + "math" + "sort" + "strconv" + + "github.com/curusarn/resh/pkg/records" +) + +// DynamicRecordDistance prediction/recommendation strategy +type DynamicRecordDistance struct { + history []records.EnrichedRecord + DistParams records.DistParams + pwdHistogram map[string]int + realPwdHistogram map[string]int + gitOriginHistogram map[string]int + MaxDepth int + Label string +} + +type strDynDistEntry struct { + cmdLine string + distance float64 +} + +// Init see name +func (s *DynamicRecordDistance) Init() { + s.history = nil + s.pwdHistogram = map[string]int{} + s.realPwdHistogram = map[string]int{} + s.gitOriginHistogram = map[string]int{} +} + +// GetTitleAndDescription see name +func (s *DynamicRecordDistance) GetTitleAndDescription() (string, string) { + return "dynamic record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use TF-IDF record distance to recommend commands" +} + +func (s *DynamicRecordDistance) idf(count int) float64 { + return math.Log(float64(len(s.history)) / float64(count)) +} + +// GetCandidates see name +func (s *DynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { + if len(s.history) == 0 { + return nil + } + var mapItems []strDynDistEntry + for i, record := range s.history { + if s.MaxDepth != 0 && i > s.MaxDepth { + break + } + distParams := records.DistParams{ + Pwd: s.DistParams.Pwd * s.idf(s.pwdHistogram[strippedRecord.PwdAfter]), + RealPwd: s.DistParams.RealPwd * s.idf(s.realPwdHistogram[strippedRecord.RealPwdAfter]), + Git: s.DistParams.Git * s.idf(s.gitOriginHistogram[strippedRecord.GitOriginRemote]), + Time: s.DistParams.Time, + SessionID: s.DistParams.SessionID, + } + distance := record.DistanceTo(strippedRecord, distParams) + mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance}) + } + sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) + var hist []string + histSet := map[string]bool{} + for _, item := range mapItems { + if histSet[item.cmdLine] { + continue + } + histSet[item.cmdLine] = true + hist = append(hist, item.cmdLine) + } + return hist +} + +// AddHistoryRecord see name +func (s *DynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { + // append record to front + s.history = append([]records.EnrichedRecord{*record}, s.history...) + s.pwdHistogram[record.Pwd]++ + s.realPwdHistogram[record.RealPwd]++ + s.gitOriginHistogram[record.GitOriginRemote]++ + return nil +} + +// ResetHistory see name +func (s *DynamicRecordDistance) ResetHistory() error { + s.Init() + return nil +} diff --git a/evaluate/strategy-frequent.go b/pkg/strat/frequent.go similarity index 52% rename from evaluate/strategy-frequent.go rename to pkg/strat/frequent.go index c41f852..ff3b912 100644 --- a/evaluate/strategy-frequent.go +++ b/pkg/strat/frequent.go @@ -1,12 +1,13 @@ -package main +package strat import ( "sort" - "github.com/curusarn/resh/common" + "github.com/curusarn/resh/pkg/records" ) -type strategyFrequent struct { +// Frequent prediction/recommendation strategy +type Frequent struct { history map[string]int } @@ -15,15 +16,18 @@ type strFrqEntry struct { count int } -func (s *strategyFrequent) init() { +// Init see name +func (s *Frequent) Init() { s.history = map[string]int{} } -func (s *strategyFrequent) GetTitleAndDescription() (string, string) { +// GetTitleAndDescription see name +func (s *Frequent) GetTitleAndDescription() (string, string) { return "frequent", "Use frequent commands" } -func (s *strategyFrequent) GetCandidates() []string { +// GetCandidates see name +func (s *Frequent) GetCandidates() []string { var mapItems []strFrqEntry for cmdLine, count := range s.history { mapItems = append(mapItems, strFrqEntry{cmdLine, count}) @@ -36,12 +40,14 @@ func (s *strategyFrequent) GetCandidates() []string { return hist } -func (s *strategyFrequent) AddHistoryRecord(record *common.Record) error { +// AddHistoryRecord see name +func (s *Frequent) AddHistoryRecord(record *records.EnrichedRecord) error { s.history[record.CmdLine]++ return nil } -func (s *strategyFrequent) ResetHistory() error { - s.history = map[string]int{} +// ResetHistory see name +func (s *Frequent) ResetHistory() error { + s.Init() return nil } diff --git a/pkg/strat/markov-chain-cmd.go b/pkg/strat/markov-chain-cmd.go new file mode 100644 index 0000000..b1fa2f5 --- /dev/null +++ b/pkg/strat/markov-chain-cmd.go @@ -0,0 +1,97 @@ +package strat + +import ( + "sort" + "strconv" + + "github.com/curusarn/resh/pkg/records" + "github.com/mb-14/gomarkov" +) + +// MarkovChainCmd prediction/recommendation strategy +type MarkovChainCmd struct { + Order int + history []strMarkCmdHistoryEntry + historyCmds []string +} + +type strMarkCmdHistoryEntry struct { + cmd string + cmdLine string +} + +type strMarkCmdEntry struct { + cmd string + transProb float64 +} + +// Init see name +func (s *MarkovChainCmd) Init() { + s.history = nil + s.historyCmds = nil +} + +// GetTitleAndDescription see name +func (s *MarkovChainCmd) GetTitleAndDescription() (string, string) { + return "command-based markov chain (order " + strconv.Itoa(s.Order) + ")", "Use command-based markov chain to recommend commands" +} + +// GetCandidates see name +func (s *MarkovChainCmd) GetCandidates() []string { + if len(s.history) < s.Order { + var hist []string + for _, item := range s.history { + hist = append(hist, item.cmdLine) + } + return hist + } + chain := gomarkov.NewChain(s.Order) + + chain.Add(s.historyCmds) + + cmdsSet := map[string]bool{} + var entries []strMarkCmdEntry + for _, cmd := range s.historyCmds { + if cmdsSet[cmd] { + continue + } + cmdsSet[cmd] = true + prob, _ := chain.TransitionProbability(cmd, s.historyCmds[len(s.historyCmds)-s.Order:]) + entries = append(entries, strMarkCmdEntry{cmd: cmd, transProb: prob}) + } + sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) + var hist []string + histSet := map[string]bool{} + for i := len(s.history) - 1; i >= 0; i-- { + if histSet[s.history[i].cmdLine] { + continue + } + histSet[s.history[i].cmdLine] = true + if s.history[i].cmd == entries[0].cmd { + hist = append(hist, s.history[i].cmdLine) + } + } + // log.Println("################") + // log.Println(s.history[len(s.history)-s.order:]) + // log.Println(" -> ") + // x := math.Min(float64(len(hist)), 3) + // log.Println(entries[:int(x)]) + // x = math.Min(float64(len(hist)), 5) + // log.Println(hist[:int(x)]) + // log.Println("################") + return hist +} + +// AddHistoryRecord see name +func (s *MarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error { + s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command}) + s.historyCmds = append(s.historyCmds, record.Command) + // s.historySet[record.CmdLine] = true + return nil +} + +// ResetHistory see name +func (s *MarkovChainCmd) ResetHistory() error { + s.Init() + return nil +} diff --git a/pkg/strat/markov-chain.go b/pkg/strat/markov-chain.go new file mode 100644 index 0000000..50c7fdc --- /dev/null +++ b/pkg/strat/markov-chain.go @@ -0,0 +1,76 @@ +package strat + +import ( + "sort" + "strconv" + + "github.com/curusarn/resh/pkg/records" + "github.com/mb-14/gomarkov" +) + +// MarkovChain prediction/recommendation strategy +type MarkovChain struct { + Order int + history []string +} + +type strMarkEntry struct { + cmdLine string + transProb float64 +} + +// Init see name +func (s *MarkovChain) Init() { + s.history = nil +} + +// GetTitleAndDescription see name +func (s *MarkovChain) GetTitleAndDescription() (string, string) { + return "markov chain (order " + strconv.Itoa(s.Order) + ")", "Use markov chain to recommend commands" +} + +// GetCandidates see name +func (s *MarkovChain) GetCandidates() []string { + if len(s.history) < s.Order { + return s.history + } + chain := gomarkov.NewChain(s.Order) + + chain.Add(s.history) + + cmdLinesSet := map[string]bool{} + var entries []strMarkEntry + for _, cmdLine := range s.history { + if cmdLinesSet[cmdLine] { + continue + } + cmdLinesSet[cmdLine] = true + prob, _ := chain.TransitionProbability(cmdLine, s.history[len(s.history)-s.Order:]) + entries = append(entries, strMarkEntry{cmdLine: cmdLine, transProb: prob}) + } + sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb }) + var hist []string + for _, item := range entries { + hist = append(hist, item.cmdLine) + } + // log.Println("################") + // log.Println(s.history[len(s.history)-s.order:]) + // log.Println(" -> ") + // x := math.Min(float64(len(hist)), 5) + // log.Println(hist[:int(x)]) + // log.Println("################") + return hist +} + +// AddHistoryRecord see name +func (s *MarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error { + s.history = append(s.history, record.CmdLine) + // s.historySet[record.CmdLine] = true + return nil +} + +// ResetHistory see name +func (s *MarkovChain) ResetHistory() error { + s.Init() + return nil +} diff --git a/pkg/strat/random.go b/pkg/strat/random.go new file mode 100644 index 0000000..0ff52f1 --- /dev/null +++ b/pkg/strat/random.go @@ -0,0 +1,57 @@ +package strat + +import ( + "math/rand" + "time" + + "github.com/curusarn/resh/pkg/records" +) + +// Random prediction/recommendation strategy +type Random struct { + CandidatesSize int + history []string + historySet map[string]bool +} + +// Init see name +func (s *Random) Init() { + s.history = nil + s.historySet = map[string]bool{} +} + +// GetTitleAndDescription see name +func (s *Random) GetTitleAndDescription() (string, string) { + return "random", "Use random commands" +} + +// GetCandidates see name +func (s *Random) GetCandidates() []string { + seed := time.Now().UnixNano() + rand.Seed(seed) + var candidates []string + candidateSet := map[string]bool{} + for len(candidates) < s.CandidatesSize && len(candidates)*2 < len(s.historySet) { + x := rand.Intn(len(s.history)) + candidate := s.history[x] + if candidateSet[candidate] == false { + candidateSet[candidate] = true + candidates = append(candidates, candidate) + continue + } + } + return candidates +} + +// AddHistoryRecord see name +func (s *Random) AddHistoryRecord(record *records.EnrichedRecord) error { + s.history = append([]string{record.CmdLine}, s.history...) + s.historySet[record.CmdLine] = true + return nil +} + +// ResetHistory see name +func (s *Random) ResetHistory() error { + s.Init() + return nil +} diff --git a/pkg/strat/recent-bash.go b/pkg/strat/recent-bash.go new file mode 100644 index 0000000..ace3571 --- /dev/null +++ b/pkg/strat/recent-bash.go @@ -0,0 +1,56 @@ +package strat + +import "github.com/curusarn/resh/pkg/records" + +// RecentBash prediction/recommendation strategy +type RecentBash struct { + histfile []string + histfileSnapshot map[string][]string + history map[string][]string +} + +// Init see name +func (s *RecentBash) Init() { + s.histfileSnapshot = map[string][]string{} + s.history = map[string][]string{} +} + +// GetTitleAndDescription see name +func (s *RecentBash) GetTitleAndDescription() (string, string) { + return "recent (bash-like)", "Behave like bash" +} + +// GetCandidates see name +func (s *RecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string { + // populate the local history from histfile + if s.histfileSnapshot[strippedRecord.SessionID] == nil { + s.histfileSnapshot[strippedRecord.SessionID] = s.histfile + } + return append(s.history[strippedRecord.SessionID], s.histfileSnapshot[strippedRecord.SessionID]...) +} + +// AddHistoryRecord see name +func (s *RecentBash) AddHistoryRecord(record *records.EnrichedRecord) error { + // remove previous occurance of record + for i, cmd := range s.history[record.SessionID] { + if cmd == record.CmdLine { + s.history[record.SessionID] = append(s.history[record.SessionID][:i], s.history[record.SessionID][i+1:]...) + } + } + // append new record + s.history[record.SessionID] = append([]string{record.CmdLine}, s.history[record.SessionID]...) + + if record.LastRecordOfSession { + // append history of the session to histfile and clear session history + s.histfile = append(s.history[record.SessionID], s.histfile...) + s.histfileSnapshot[record.SessionID] = nil + s.history[record.SessionID] = nil + } + return nil +} + +// ResetHistory see name +func (s *RecentBash) ResetHistory() error { + s.Init() + return nil +} diff --git a/pkg/strat/recent.go b/pkg/strat/recent.go new file mode 100644 index 0000000..157b52c --- /dev/null +++ b/pkg/strat/recent.go @@ -0,0 +1,37 @@ +package strat + +import "github.com/curusarn/resh/pkg/records" + +// Recent prediction/recommendation strategy +type Recent struct { + history []string +} + +// GetTitleAndDescription see name +func (s *Recent) GetTitleAndDescription() (string, string) { + return "recent", "Use recent commands" +} + +// GetCandidates see name +func (s *Recent) GetCandidates() []string { + return s.history +} + +// AddHistoryRecord see name +func (s *Recent) AddHistoryRecord(record *records.EnrichedRecord) error { + // remove previous occurance of record + for i, cmd := range s.history { + if cmd == record.CmdLine { + s.history = append(s.history[:i], s.history[i+1:]...) + } + } + // append new record + s.history = append([]string{record.CmdLine}, s.history...) + return nil +} + +// ResetHistory see name +func (s *Recent) ResetHistory() error { + s.history = nil + return nil +} diff --git a/pkg/strat/record-distance.go b/pkg/strat/record-distance.go new file mode 100644 index 0000000..e582584 --- /dev/null +++ b/pkg/strat/record-distance.go @@ -0,0 +1,70 @@ +package strat + +import ( + "sort" + "strconv" + + "github.com/curusarn/resh/pkg/records" +) + +// RecordDistance prediction/recommendation strategy +type RecordDistance struct { + history []records.EnrichedRecord + DistParams records.DistParams + MaxDepth int + Label string +} + +type strDistEntry struct { + cmdLine string + distance float64 +} + +// Init see name +func (s *RecordDistance) Init() { + s.history = nil +} + +// GetTitleAndDescription see name +func (s *RecordDistance) GetTitleAndDescription() (string, string) { + return "record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use record distance to recommend commands" +} + +// GetCandidates see name +func (s *RecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string { + if len(s.history) == 0 { + return nil + } + var mapItems []strDistEntry + for i, record := range s.history { + if s.MaxDepth != 0 && i > s.MaxDepth { + break + } + distance := record.DistanceTo(strippedRecord, s.DistParams) + mapItems = append(mapItems, strDistEntry{record.CmdLine, distance}) + } + sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance }) + var hist []string + histSet := map[string]bool{} + for _, item := range mapItems { + if histSet[item.cmdLine] { + continue + } + histSet[item.cmdLine] = true + hist = append(hist, item.cmdLine) + } + return hist +} + +// AddHistoryRecord see name +func (s *RecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error { + // append record to front + s.history = append([]records.EnrichedRecord{*record}, s.history...) + return nil +} + +// ResetHistory see name +func (s *RecordDistance) ResetHistory() error { + s.Init() + return nil +} diff --git a/pkg/strat/strat.go b/pkg/strat/strat.go new file mode 100644 index 0000000..28ac015 --- /dev/null +++ b/pkg/strat/strat.go @@ -0,0 +1,46 @@ +package strat + +import ( + "github.com/curusarn/resh/pkg/records" +) + +// ISimpleStrategy interface +type ISimpleStrategy interface { + GetTitleAndDescription() (string, string) + GetCandidates() []string + AddHistoryRecord(record *records.EnrichedRecord) error + ResetHistory() error +} + +// IStrategy interface +type IStrategy interface { + GetTitleAndDescription() (string, string) + GetCandidates(r records.EnrichedRecord) []string + AddHistoryRecord(record *records.EnrichedRecord) error + ResetHistory() error +} + +type simpleStrategyWrapper struct { + strategy ISimpleStrategy +} + +// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy +func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper { + return &simpleStrategyWrapper{strategy: strategy} +} + +func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) { + return s.strategy.GetTitleAndDescription() +} + +func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string { + return s.strategy.GetCandidates() +} + +func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error { + return s.strategy.AddHistoryRecord(r) +} + +func (s *simpleStrategyWrapper) ResetHistory() error { + return s.strategy.ResetHistory() +} diff --git a/install_helper.sh b/scripts/install_helper.sh similarity index 100% rename from install_helper.sh rename to scripts/install_helper.sh diff --git a/rawinstall.sh b/scripts/rawinstall.sh similarity index 100% rename from rawinstall.sh rename to scripts/rawinstall.sh diff --git a/evaluate/resh-evaluate-plot.py b/scripts/resh-evaluate-plot.py similarity index 79% rename from evaluate/resh-evaluate-plot.py rename to scripts/resh-evaluate-plot.py index 45d9322..4ffea19 100755 --- a/evaluate/resh-evaluate-plot.py +++ b/scripts/resh-evaluate-plot.py @@ -9,6 +9,7 @@ import matplotlib.pyplot as plt import matplotlib.path as mpath import numpy as np from graphviz import Digraph +from datetime import datetime PLOT_WIDTH = 10 # inches PLOT_HEIGHT = 7 # inches @@ -22,11 +23,11 @@ DATA_records_by_session = defaultdict(list) for user in data["UsersRecords"]: for device in user["Devices"]: for record in device["Records"]: - if record["invalid"]: + if "invalid" in record and record["invalid"]: continue DATA_records.append(record) - DATA_records_by_session[record["sessionId"]].append(record) + DATA_records_by_session[record["seqSessionId"]].append(record) DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"])) @@ -39,7 +40,6 @@ async_draw = True # for strategy in data["Strategies"]: # print(json.dumps(strategy)) - def zipf(length): return list(map(lambda x: 1/2**x, range(0, length))) @@ -81,7 +81,7 @@ def plot_cmdLineFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): def plot_cmdFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False): cmd_count = defaultdict(int) for record in DATA_records: - cmd = record["firstWord"] + cmd = record["command"] if cmd == "": continue cmd_count[cmd] += 1 @@ -111,7 +111,7 @@ def plot_cmdVocabularySize_cmdLinesEntered(): cmd_vocabulary = set() y_cmd_count = [0] for record in DATA_records: - cmd = record["firstWord"] + cmd = record["command"] if cmd in cmd_vocabulary: # repeat last value y_cmd_count.append(y_cmd_count[-1]) @@ -163,7 +163,7 @@ def plot_cmdLineVocabularySize_cmdLinesEntered(): # Figure 3.3. Sequential structure of UNIX command usage, from Figure 4 in Hanson et al. (1984). # Ball diameters are proportional to stationary probability. Lines indicate significant dependencies, # solid ones being more probable (p < .0001) and dashed ones less probable (.005 < p < .0001). -def graph_cmdSequences(node_count=33, edge_minValue=0.05): +def graph_cmdSequences(node_count=33, edge_minValue=0.05, view_graph=True): START_CMD = "_start_" cmd_count = defaultdict(int) cmdSeq_count = defaultdict(lambda: defaultdict(int)) @@ -174,7 +174,7 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05): cmd_count[START_CMD] += 1 prev_cmd = START_CMD for record in session: - cmd = record["firstWord"] + cmd = record["command"] cmdSeq_count[prev_cmd][cmd] += 1 cmd_count[cmd] += 1 if cmd not in cmd_id: @@ -265,8 +265,8 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05): # graphviz sometimes fails - see above try: - graph.view() - # graph.render('/tmp/resh-graphviz-cmdSeq.gv', view=True) + # graph.view() + graph.render('/tmp/resh-graph-command_sequence-nodeCount_{}-edgeMinVal_{}.gv'.format(node_count, edge_minValue), view=view_graph) break except Exception as e: trace = traceback.format_exc() @@ -275,7 +275,7 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05): def plot_strategies_matches(plot_size=50, selected_strategies=[]): plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Matches at distance") + plt.title("Matches at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) plt.ylabel('%' + " of matches") plt.xlabel("Distance") legend = [] @@ -348,10 +348,9 @@ def plot_strategies_matches(plot_size=50, selected_strategies=[]): plt.show() - def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) - plt.title("Average characters recalled at distance") + plt.title("Average characters recalled at distance <{}>".format(datetime.now().strftime('%H:%M:%S'))) plt.ylabel("Average characters recalled") plt.xlabel("Distance") x_values = range(1, plot_size+1) @@ -420,19 +419,101 @@ def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]): plt.show() +def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]): + plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT)) + plt.title("Average characters recalled at distance (including prefix matches) <{}>".format(datetime.now().strftime('%H:%M:%S'))) + plt.ylabel("Average characters recalled (including prefix matches)") + plt.xlabel("Distance") + x_values = range(1, plot_size+1) + legend = [] + saved_charsRecalled_total = None + saved_dataPoint_count = None + for strategy in data["Strategies"]: + strategy_title = strategy["Title"] + # strategy_description = strategy["Description"] + + dataPoint_count = 0 + matches_total = 0 + charsRecalled = [0] * plot_size + charsRecalled_total = 0 -# graph_cmdSequences(node_count=33, edge_minValue=0.05) -graph_cmdSequences(node_count=28, edge_minValue=0.06) + for multiMatch in strategy["PrefixMatches"]: + dataPoint_count += 1 + + if not multiMatch["Match"]: + continue + matches_total += 1 + + last_charsRecalled = 0 + for match in multiMatch["Entries"]: + + chars = match["CharsRecalled"] + charsIncrease = chars - last_charsRecalled + assert(charsIncrease > 0) + charsRecalled_total += charsIncrease + + dist = match["Distance"] + if dist > plot_size: + continue + + charsRecalled[dist-1] += charsIncrease + last_charsRecalled = chars + + # recent is very simple strategy so we will believe + # that there is no bug in it and we can use it to determine total + if strategy_title == "recent": + saved_charsRecalled_total = charsRecalled_total + saved_dataPoint_count = dataPoint_count + + if len(selected_strategies) and strategy_title not in selected_strategies: + continue + + acc = 0 + charsRecalled_cumulative = [] + for x in charsRecalled: + acc += x + charsRecalled_cumulative.append(acc) + charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative)) + + plt.plot(x_values, charsRecalled_average, 'o-') + legend.append(strategy_title) + + assert(saved_charsRecalled_total is not None) + assert(saved_dataPoint_count is not None) + max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values) + plt.plot(x_values, max_values, 'r-') + legend.append("maximum possible") + + x_ticks = list(range(1, plot_size+1, 2)) + x_labels = x_ticks[:] + plt.xticks(x_ticks, x_labels) + plt.legend(legend, loc="best") + if async_draw: + plt.draw() + else: + plt.show() + -plot_cmdLineFrq_rank() -plot_cmdFrq_rank() +# plot_cmdLineFrq_rank() +# plot_cmdFrq_rank() -plot_cmdLineVocabularySize_cmdLinesEntered() -plot_cmdVocabularySize_cmdLinesEntered() +# plot_cmdLineVocabularySize_cmdLinesEntered() +# plot_cmdVocabularySize_cmdLinesEntered() plot_strategies_matches(20) plot_strategies_charsRecalled(20) +# plot_strategies_charsRecalled_prefix(20) + +# graph_cmdSequences(node_count=33, edge_minValue=0.048) + +# graph_cmdSequences(node_count=28, edge_minValue=0.06) + +# for n in range(29, 35): +# for e in range(44, 56, 2): +# e *= 0.001 +# graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False) + +# be careful and check if labels fit the display if async_draw: plt.show() -# be careful and check if labels fit the display \ No newline at end of file diff --git a/shellrc.sh b/scripts/shellrc.sh similarity index 100% rename from shellrc.sh rename to scripts/shellrc.sh diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..3eca974 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +[ "${BASH_SOURCE[0]}" != "scripts/test.sh" ] && echo 'Run this script using `make test`' && exit 1 + +for f in scripts/*.sh; do + echo "Running shellcheck on $f ..." + shellcheck $f --shell=bash --severity=error || exit 1 +done + +echo "Checking Zsh syntax of scripts/shellrc.sh ..." +! zsh -n scripts/shellrc.sh && echo "Zsh syntax check failed!" && exit 1 + +for sh in bash zsh; do + echo "Running functions in scripts/shellrc.sh using $sh ..." + ! $sh -c ". scripts/shellrc.sh; __resh_preexec; __resh_precmd" && echo "Error while running functions!" && exit 1 +done + +# TODO: test installation + +exit 0 diff --git a/uuid.sh b/scripts/uuid.sh similarity index 100% rename from uuid.sh rename to scripts/uuid.sh