Merge pull request #15 from curusarn/dev_3

Restructure the project, improve evaluation, add tests (go, shell)
6 years ago · e4e1ac3e7a
parent 0cd433ba5e 29bc4b20b8
commit e4e1ac3e7a
40 changed files with 1987 additions and 793 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1 @@
-resh-collect
-resh-daemon
-resh-sanitize-history
-resh-evaluate
+bin/*
--- a/47
+++ b/47
@ -1,10 +1,10 @@
 SHELL=/bin/bash
-VERSION=$(shell cat version)
+VERSION=$(shell cat VERSION)
 REVISION=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_revision")
 GOFLAGS=-ldflags "-X main.Version=${VERSION} -X main.Revision=${REVISION}"

 autoinstall: 
-	./install_helper.sh
+	scripts/install_helper.sh

 sanitize:
 	#
@ -23,8 +23,8 @@ sanitize:
 	#
 	#
 	# Running history sanitization ...
-	resh-sanitize-history -trim-hashes 0 --output ~/resh_history_sanitized.json
-	resh-sanitize-history -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json
+	resh-sanitize -trim-hashes 0 --output ~/resh_history_sanitized.json
+	resh-sanitize -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json
 	# 
 	# 
 	# SUCCESS - ALL DONE!
@ -41,8 +41,17 @@ sanitize:
 	#
 	#

+build: test_go submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize

-build: submodules resh-collect resh-daemon resh-sanitize-history resh-evaluate
+test_go:
+	# Running tests
+	@for dir in {cmd,pkg}/* ; do \
+		echo $$dir ; \
+		go test $$dir/*.go ; \
+	done
+
+test:
+	scripts/test.sh

 rebuild:
 	make clean
@ -51,15 +60,15 @@ rebuild:
 clean:
 	rm resh-*

-install: build submodules/bash-preexec/bash-preexec.sh shellrc.sh config.toml uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config
+install: build submodules/bash-preexec/bash-preexec.sh scripts/shellrc.sh conf/config.toml scripts/uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config
 	# Copying files to resh directory ...
 	cp -f submodules/bash-preexec/bash-preexec.sh ~/.bash-preexec.sh
-	cp -f config.toml ~/.config/resh.toml
-	cp -f shellrc.sh ~/.resh/shellrc
-	cp -f uuid.sh ~/.resh/bin/resh-uuid
-	cp -f resh-* ~/.resh/bin/
-	cp -f evaluate/resh-evaluate-plot.py ~/.resh/bin/
-	cp -fr sanitizer_data ~/.resh/
+	cp -f conf/config.toml ~/.config/resh.toml
+	cp -f scripts/shellrc.sh ~/.resh/shellrc
+	cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid
+	cp -f bin/* ~/.resh/bin/
+	cp -f scripts/resh-evaluate-plot.py ~/.resh/bin/
+	cp -fr data/sanitizer ~/.resh/
 	# backward compatibility: We have a new location for resh history file 
 	[ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json 
 	# Adding resh shellrc to .bashrc ...
@ -107,17 +116,8 @@ uninstall:
 	# Uninstalling ...
 	-rm -rf ~/.resh/

-resh-daemon: daemon/resh-daemon.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $<
-
-resh-collect: collect/resh-collect.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $<
-
-resh-sanitize-history: sanitize-history/resh-sanitize-history.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $<
-
-resh-evaluate: evaluate/resh-evaluate.go evaluate/strategy-*.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $< evaluate/strategy-*.go 
+bin/resh-%: cmd/%/main.go pkg/*/*.go VERSION
+	go build ${GOFLAGS} -o $@ cmd/$*/*.go

 $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config:
 	# Creating dirs ...
@ -129,7 +129,6 @@ $(HOME)/.resh/resh-uuid:

 .PHONY: submodules build install rebuild uninstall clean autoinstall

-
 submodules: | submodules/bash-preexec/bash-preexec.sh
 	@# sets submodule.recurse to true if unset
 	@# sets status.submoduleSummary to true if unset
--- a/README.md
+++ b/README.md
@ -17,6 +17,7 @@ If you are not happy with it you can uninstall it with a single command (`rm -rf
 The ultimate point of my thesis is to provide a context-based replacement/enhancement for bash and zsh shell history.

 The idea is to:
+
 - Save each command with metadata (device, directory, git, time, terminal session pid, ... see example below)
 - Recommend history based on saved metadata
  - e.g. it will be easier to get to commands specific to the project you are currently working on (based on directory, git repository url, ...)
@ -45,9 +46,11 @@ If you install RESH, please give me some contact info using this form: https://f
 ## Installation

 ### Simplest
-Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/rawinstall.sh)"` from anywhere.
+
+Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/scripts/rawinstall.sh)"` from anywhere.

 ### Simple
+
 1. Run `git clone https://github.com/curusarn/resh.git && cd resh`
 2. Run `make autoinstall` for assisted build & instalation.
    - OR Run `make install` if you know how to build Golang projects.
@ -59,6 +62,7 @@ If you install RESH, please give me some contact info using this form: https://f
 Works in `bash` and `zsh`.

 Tested on:
+
 - Arch
 - MacOS
 - Ubuntu (18.04)
--- a/0
+++ b/0
--- a/collect/resh-collect.go
+++ b/collect/resh-collect.go
@ -11,7 +11,8 @@ import (
 	"os"

 	"github.com/BurntSushi/toml"
-	common "github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/cfg"
+	"github.com/curusarn/resh/pkg/records"

 	//  "os/exec"
 	"os/user"
@ -30,11 +31,11 @@ func main() {
 	usr, _ := user.Current()
 	dir := usr.HomeDir
 	configPath := filepath.Join(dir, "/.config/resh.toml")
-	reshUuidPath := filepath.Join(dir, "/.resh/resh-uuid")
+	reshUUIDPath := filepath.Join(dir, "/.resh/resh-uuid")

-	machineIdPath := "/etc/machine-id"
+	machineIDPath := "/etc/machine-id"

-	var config common.Config
+	var config cfg.Config
 	if _, err := toml.DecodeFile(configPath, &config); err != nil {
 		log.Fatal("Error reading config:", err)
 	}
@ -48,7 +49,7 @@ func main() {
 	exitCode := flag.Int("exitCode", -1, "exit code")
 	shell := flag.String("shell", "", "actual shell")
 	uname := flag.String("uname", "", "uname")
-	sessionId := flag.String("sessionId", "", "resh generated session id")
+	sessionID := flag.String("sessionId", "", "resh generated session id")

 	// posix variables
 	cols := flag.String("cols", "-1", "$COLUMNS")
@ -82,10 +83,10 @@ func main() {
 	timezoneBefore := flag.String("timezoneBefore", "", "")
 	timezoneAfter := flag.String("timezoneAfter", "", "")

-	osReleaseId := flag.String("osReleaseId", "", "/etc/os-release ID")
-	osReleaseVersionId := flag.String("osReleaseVersionId", "",
+	osReleaseID := flag.String("osReleaseId", "", "/etc/os-release ID")
+	osReleaseVersionID := flag.String("osReleaseVersionId", "",
 		"/etc/os-release ID")
-	osReleaseIdLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID")
+	osReleaseIDLike := flag.String("osReleaseIdLike", "", "/etc/os-release ID")
 	osReleaseName := flag.String("osReleaseName", "", "/etc/os-release ID")
 	osReleasePrettyName := flag.String("osReleasePrettyName", "",
 		"/etc/os-release ID")
@ -161,8 +162,8 @@ func main() {
 		*gitRemote = ""
 	}

-	if *osReleaseId == "" {
-		*osReleaseId = "linux"
+	if *osReleaseID == "" {
+		*osReleaseID = "linux"
 	}
 	if *osReleaseName == "" {
 		*osReleaseName = "Linux"
@ -171,78 +172,80 @@ func main() {
 		*osReleasePrettyName = "Linux"
 	}

-	rec := common.Record{
-		// core
-		CmdLine:   *cmdLine,
-		ExitCode:  *exitCode,
-		Shell:     *shell,
-		Uname:     *uname,
-		SessionId: *sessionId,
-
+	rec := records.Record{
 		// posix
 		Cols:  *cols,
 		Lines: *lines,
-
-		Home:  *home,
-		Lang:  *lang,
-		LcAll: *lcAll,
-		Login: *login,
-		// Path:     *path,
-		Pwd:      *pwd,
-		PwdAfter: *pwdAfter,
-		ShellEnv: *shellEnv,
-		Term:     *term,
-
-		// non-posix
-		RealPwd:      realPwd,
-		RealPwdAfter: realPwdAfter,
-		Pid:          *pid,
-		SessionPid:   *sessionPid,
-		Host:         *host,
-		Hosttype:     *hosttype,
-		Ostype:       *ostype,
-		Machtype:     *machtype,
-		Shlvl:        *shlvl,
-
-		// before after
-		TimezoneBefore: *timezoneBefore,
-		TimezoneAfter:  *timezoneAfter,
-
-		RealtimeBefore:      realtimeBefore,
-		RealtimeAfter:       realtimeAfter,
-		RealtimeBeforeLocal: realtimeBeforeLocal,
-		RealtimeAfterLocal:  realtimeAfterLocal,
-
-		RealtimeDuration:          realtimeDuration,
-		RealtimeSinceSessionStart: realtimeSinceSessionStart,
-		RealtimeSinceBoot:         realtimeSinceBoot,
-
-		GitDir:          gitDir,
-		GitRealDir:      gitRealDir,
-		GitOriginRemote: *gitRemote,
-		MachineId:       readFileContent(machineIdPath),
-
-		OsReleaseId:         *osReleaseId,
-		OsReleaseVersionId:  *osReleaseVersionId,
-		OsReleaseIdLike:     *osReleaseIdLike,
-		OsReleaseName:       *osReleaseName,
-		OsReleasePrettyName: *osReleasePrettyName,
-
-		ReshUuid:     readFileContent(reshUuidPath),
-		ReshVersion:  Version,
-		ReshRevision: Revision,
+		// core
+		BaseRecord: records.BaseRecord{
+			CmdLine:   *cmdLine,
+			ExitCode:  *exitCode,
+			Shell:     *shell,
+			Uname:     *uname,
+			SessionID: *sessionID,
+
+			// posix
+			Home:  *home,
+			Lang:  *lang,
+			LcAll: *lcAll,
+			Login: *login,
+			// Path:     *path,
+			Pwd:      *pwd,
+			PwdAfter: *pwdAfter,
+			ShellEnv: *shellEnv,
+			Term:     *term,
+
+			// non-posix
+			RealPwd:      realPwd,
+			RealPwdAfter: realPwdAfter,
+			Pid:          *pid,
+			SessionPid:   *sessionPid,
+			Host:         *host,
+			Hosttype:     *hosttype,
+			Ostype:       *ostype,
+			Machtype:     *machtype,
+			Shlvl:        *shlvl,
+
+			// before after
+			TimezoneBefore: *timezoneBefore,
+			TimezoneAfter:  *timezoneAfter,
+
+			RealtimeBefore:      realtimeBefore,
+			RealtimeAfter:       realtimeAfter,
+			RealtimeBeforeLocal: realtimeBeforeLocal,
+			RealtimeAfterLocal:  realtimeAfterLocal,
+
+			RealtimeDuration:          realtimeDuration,
+			RealtimeSinceSessionStart: realtimeSinceSessionStart,
+			RealtimeSinceBoot:         realtimeSinceBoot,
+
+			GitDir:          gitDir,
+			GitRealDir:      gitRealDir,
+			GitOriginRemote: *gitRemote,
+			MachineID:       readFileContent(machineIDPath),
+
+			OsReleaseID:         *osReleaseID,
+			OsReleaseVersionID:  *osReleaseVersionID,
+			OsReleaseIDLike:     *osReleaseIDLike,
+			OsReleaseName:       *osReleaseName,
+			OsReleasePrettyName: *osReleasePrettyName,
+
+			ReshUUID:     readFileContent(reshUUIDPath),
+			ReshVersion:  Version,
+			ReshRevision: Revision,
+		},
 	}
 	sendRecord(rec, strconv.Itoa(config.Port))
 }

-func sendRecord(r common.Record, port string) {
-	recJson, err := json.Marshal(r)
+func sendRecord(r records.Record, port string) {
+	recJSON, err := json.Marshal(r)
 	if err != nil {
 		log.Fatal("send err 1", err)
 	}

 	req, err := http.NewRequest("POST", "http://localhost:"+port+"/record",
-		bytes.NewBuffer(recJson))
+		bytes.NewBuffer(recJSON))
 	if err != nil {
 		log.Fatal("send err 2", err)
 	}
@ -279,14 +282,14 @@ func getGitDirs(cdup string, exitCode int, pwd string) (string, string) {

 func getTimezoneOffsetInSeconds(zone string) float64 {
 	// date +%z -> "+0200"
-	hours_str := zone[:3]
-	mins_str := zone[3:]
-	hours, err := strconv.Atoi(hours_str)
+	hoursStr := zone[:3]
+	minsStr := zone[3:]
+	hours, err := strconv.Atoi(hoursStr)
 	if err != nil {
 		log.Println("err while parsing hours in timezone offset:", err)
 		return -1
 	}
-	mins, err := strconv.Atoi(mins_str)
+	mins, err := strconv.Atoi(minsStr)
 	if err != nil {
 		log.Println("err while parsing mins in timezone offset:", err)
 		return -1
--- a/daemon/resh-daemon.go
+++ b/daemon/resh-daemon.go
@ -14,7 +14,8 @@ import (
 	"strings"

 	"github.com/BurntSushi/toml"
-	common "github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/cfg"
+	"github.com/curusarn/resh/pkg/records"
 )

 // Version from git set during build
@ -43,7 +44,7 @@ func main() {
 	log.SetOutput(f)
 	log.SetPrefix(strconv.Itoa(os.Getpid()) + " | ")

-	var config common.Config
+	var config cfg.Config
 	if _, err := toml.DecodeFile(configPath, &config); err != nil {
 		log.Println("Error reading config", err)
 		return
@ -88,7 +89,7 @@ type recordHandler struct {

 func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("OK\n"))
-	record := common.Record{}
+	record := records.Record{}

 	jsn, err := ioutil.ReadAll(r.Body)
 	if err != nil {
--- a/cmd/evaluate/main.go
+++ b/cmd/evaluate/main.go
@ -0,0 +1,152 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/user"
+	"path/filepath"
+
+	"github.com/curusarn/resh/pkg/histanal"
+	"github.com/curusarn/resh/pkg/records"
+	"github.com/curusarn/resh/pkg/strat"
+)
+
+// Version from git set during build
+var Version string
+
+// Revision from git set during build
+var Revision string
+
+func main() {
+	const maxCandidates = 50
+
+	usr, _ := user.Current()
+	dir := usr.HomeDir
+	historyPath := filepath.Join(dir, ".resh_history.json")
+	historyPathBatchMode := filepath.Join(dir, "resh_history.json")
+	sanitizedHistoryPath := filepath.Join(dir, "resh_history_sanitized.json")
+	// tmpPath := "/tmp/resh-evaluate-tmp.json"
+
+	showVersion := flag.Bool("version", false, "Show version and exit")
+	showRevision := flag.Bool("revision", false, "Show git revision and exit")
+	input := flag.String("input", "",
+		"Input file (default: "+historyPath+"OR"+sanitizedHistoryPath+
+			" depending on --sanitized-input option)")
+	// outputDir := flag.String("output", "/tmp/resh-evaluate", "Output directory")
+	sanitizedInput := flag.Bool("sanitized-input", false,
+		"Handle input as sanitized (also changes default value for input argument)")
+	plottingScript := flag.String("plotting-script", "resh-evaluate-plot.py", "Script to use for plotting")
+	inputDataRoot := flag.String("input-data-root", "",
+		"Input data root, enables batch mode, looks for files matching --input option")
+	slow := flag.Bool("slow", false,
+		"Enables strategies that takes a long time (e.g. markov chain strategies).")
+	skipFailedCmds := flag.Bool("skip-failed-cmds", false,
+		"Skips records with non-zero exit status.")
+	debugRecords := flag.Float64("debug", 0, "Debug records - percentage of records that should be debugged.")
+
+	flag.Parse()
+
+	// handle show{Version,Revision} options
+	if *showVersion == true {
+		fmt.Println(Version)
+		os.Exit(0)
+	}
+	if *showRevision == true {
+		fmt.Println(Revision)
+		os.Exit(0)
+	}
+
+	// handle batch mode
+	batchMode := false
+	if *inputDataRoot != "" {
+		batchMode = true
+	}
+	// set default input
+	if *input == "" {
+		if *sanitizedInput {
+			*input = sanitizedHistoryPath
+		} else if batchMode {
+			*input = historyPathBatchMode
+		} else {
+			*input = historyPath
+		}
+	}
+
+	var evaluator histanal.HistEval
+	if batchMode {
+		evaluator = histanal.NewHistEvalBatchMode(*input, *inputDataRoot, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput)
+	} else {
+		evaluator = histanal.NewHistEval(*input, maxCandidates, *skipFailedCmds, *debugRecords, *sanitizedInput)
+	}
+
+	var simpleStrategies []strat.ISimpleStrategy
+	var strategies []strat.IStrategy
+
+	// dummy := strategyDummy{}
+	// simpleStrategies = append(simpleStrategies, &dummy)
+
+	simpleStrategies = append(simpleStrategies, &strat.Recent{})
+
+	// frequent := strategyFrequent{}
+	// frequent.init()
+	// simpleStrategies = append(simpleStrategies, &frequent)
+
+	// random := strategyRandom{candidatesSize: maxCandidates}
+	// random.init()
+	// simpleStrategies = append(simpleStrategies, &random)
+
+	directory := strat.DirectorySensitive{}
+	directory.Init()
+	simpleStrategies = append(simpleStrategies, &directory)
+
+	dynamicDistG := strat.DynamicRecordDistance{
+		MaxDepth:   3000,
+		DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1, Git: 10},
+		Label:      "10*pwd,10*realpwd,session,time,10*git",
+	}
+	dynamicDistG.Init()
+	strategies = append(strategies, &dynamicDistG)
+
+	distanceStaticBest := strat.RecordDistance{
+		MaxDepth:   3000,
+		DistParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1},
+		Label:      "10*pwd,10*realpwd,session,time",
+	}
+	strategies = append(strategies, &distanceStaticBest)
+
+	recentBash := strat.RecentBash{}
+	recentBash.Init()
+	strategies = append(strategies, &recentBash)
+
+	if *slow {
+
+		markovCmd := strat.MarkovChainCmd{Order: 1}
+		markovCmd.Init()
+
+		markovCmd2 := strat.MarkovChainCmd{Order: 2}
+		markovCmd2.Init()
+
+		markov := strat.MarkovChain{Order: 1}
+		markov.Init()
+
+		markov2 := strat.MarkovChain{Order: 2}
+		markov2.Init()
+
+		simpleStrategies = append(simpleStrategies, &markovCmd2, &markovCmd, &markov2, &markov)
+	}
+
+	for _, strategy := range simpleStrategies {
+		strategies = append(strategies, strat.NewSimpleStrategyWrapper(strategy))
+	}
+
+	for _, strat := range strategies {
+		err := evaluator.Evaluate(strat)
+		if err != nil {
+			log.Println("Evaluator evaluate() error:", err)
+		}
+	}
+
+	evaluator.CalculateStatsAndPlot(*plottingScript)
+}
--- a/sanitize-history/resh-sanitize-history.go
+++ b/sanitize-history/resh-sanitize-history.go
@ -19,7 +19,7 @@ import (
 	"strings"
 	"unicode"

-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 	giturls "github.com/whilp/git-urls"
 )

@ -79,8 +79,8 @@ func main() {

 	scanner := bufio.NewScanner(inputFile)
 	for scanner.Scan() {
-		record := common.Record{}
-		fallbackRecord := common.FallbackRecord{}
+		record := records.Record{}
+		fallbackRecord := records.FallbackRecord{}
 		line := scanner.Text()
 		err = json.Unmarshal([]byte(line), &record)
 		if err != nil {
@ -89,7 +89,7 @@ func main() {
 				log.Println("Line:", line)
 				log.Fatal("Decoding error:", err)
 			}
-			record = common.ConvertRecord(&fallbackRecord)
+			record = records.ConvertRecord(&fallbackRecord)
 		}
 		err = sanitizer.sanitizeRecord(&record)
 		if err != nil {
@ -139,7 +139,7 @@ func loadData(fname string) map[string]bool {
 	return data
 }

-func (s *sanitizer) sanitizeRecord(record *common.Record) error {
+func (s *sanitizer) sanitizeRecord(record *records.Record) error {
 	// hash directories of the paths
 	record.Pwd = s.sanitizePath(record.Pwd)
 	record.RealPwd = s.sanitizePath(record.RealPwd)
@ -153,7 +153,7 @@ func (s *sanitizer) sanitizeRecord(record *common.Record) error {
 	// hash the most sensitive info, do not tokenize
 	record.Host = s.hashToken(record.Host)
 	record.Login = s.hashToken(record.Login)
-	record.MachineId = s.hashToken(record.MachineId)
+	record.MachineID = s.hashToken(record.MachineID)

 	var err error
 	// this changes git url a bit but I'm still happy with the result
--- a/common/resh-common.go
+++ b/common/resh-common.go
@ -1,247 +0,0 @@
-package common
-
-import (
-	"log"
-	"strconv"
-
-	"github.com/mattn/go-shellwords"
-)
-
-// Record representing single executed command with its metadata
-type Record struct {
-	// core
-	CmdLine   string `json:"cmdLine"`
-	ExitCode  int    `json:"exitCode"`
-	Shell     string `json:"shell"`
-	Uname     string `json:"uname"`
-	SessionId string `json:"sessionId"`
-
-	// posix
-	Cols  string `json:"cols"`
-	Lines string `json:"lines"`
-	Home  string `json:"home"`
-	Lang  string `json:"lang"`
-	LcAll string `json:"lcAll"`
-	Login string `json:"login"`
-	//Path     string `json:"path"`
-	Pwd      string `json:"pwd"`
-	PwdAfter string `json:"pwdAfter"`
-	ShellEnv string `json:"shellEnv"`
-	Term     string `json:"term"`
-
-	// non-posix"`
-	RealPwd      string `json:"realPwd"`
-	RealPwdAfter string `json:"realPwdAfter"`
-	Pid          int    `json:"pid"`
-	SessionPid   int    `json:"sessionPid"`
-	Host         string `json:"host"`
-	Hosttype     string `json:"hosttype"`
-	Ostype       string `json:"ostype"`
-	Machtype     string `json:"machtype"`
-	Shlvl        int    `json:"shlvl"`
-
-	// before after
-	TimezoneBefore string `json:"timezoneBefore"`
-	TimezoneAfter  string `json:"timezoneAfter"`
-
-	RealtimeBefore      float64 `json:"realtimeBefore"`
-	RealtimeAfter       float64 `json:"realtimeAfter"`
-	RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"`
-	RealtimeAfterLocal  float64 `json:"realtimeAfterLocal"`
-
-	RealtimeDuration          float64 `json:"realtimeDuration"`
-	RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"`
-	RealtimeSinceBoot         float64 `json:"realtimeSinceBoot"`
-	//Logs []string      `json: "logs"`
-
-	GitDir          string `json:"gitDir"`
-	GitRealDir      string `json:"gitRealDir"`
-	GitOriginRemote string `json:"gitOriginRemote"`
-	MachineId       string `json:"machineId"`
-
-	OsReleaseId         string `json:"osReleaseId"`
-	OsReleaseVersionId  string `json:"osReleaseVersionId"`
-	OsReleaseIdLike     string `json:"osReleaseIdLike"`
-	OsReleaseName       string `json:"osReleaseName"`
-	OsReleasePrettyName string `json:"osReleasePrettyName"`
-
-	ReshUuid     string `json:"reshUuid"`
-	ReshVersion  string `json:"reshVersion"`
-	ReshRevision string `json:"reshRevision"`
-
-	// added by sanitizatizer
-	Sanitized bool `json:"sanitized"`
-	CmdLength int  `json:"cmdLength,omitempty"`
-
-	// enriching fields - added "later"
-	FirstWord    string `json:"firstWord,omitempty"`
-	Invalid      bool   `json:"invalid,omitempty"`
-	SeqSessionID uint64 `json:"seqSessionID,omitempty"`
-}
-
-// FallbackRecord when record is too old and can't be parsed into regular Record
-type FallbackRecord struct {
-	// older version of the record where cols and lines are int
-
-	// core
-	CmdLine   string `json:"cmdLine"`
-	ExitCode  int    `json:"exitCode"`
-	Shell     string `json:"shell"`
-	Uname     string `json:"uname"`
-	SessionId string `json:"sessionId"`
-
-	// posix
-	Cols  int    `json:"cols"`  // notice the in type
-	Lines int    `json:"lines"` // notice the in type
-	Home  string `json:"home"`
-	Lang  string `json:"lang"`
-	LcAll string `json:"lcAll"`
-	Login string `json:"login"`
-	//Path     string `json:"path"`
-	Pwd      string `json:"pwd"`
-	PwdAfter string `json:"pwdAfter"`
-	ShellEnv string `json:"shellEnv"`
-	Term     string `json:"term"`
-
-	// non-posix"`
-	RealPwd      string `json:"realPwd"`
-	RealPwdAfter string `json:"realPwdAfter"`
-	Pid          int    `json:"pid"`
-	SessionPid   int    `json:"sessionPid"`
-	Host         string `json:"host"`
-	Hosttype     string `json:"hosttype"`
-	Ostype       string `json:"ostype"`
-	Machtype     string `json:"machtype"`
-	Shlvl        int    `json:"shlvl"`
-
-	// before after
-	TimezoneBefore string `json:"timezoneBefore"`
-	TimezoneAfter  string `json:"timezoneAfter"`
-
-	RealtimeBefore      float64 `json:"realtimeBefore"`
-	RealtimeAfter       float64 `json:"realtimeAfter"`
-	RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"`
-	RealtimeAfterLocal  float64 `json:"realtimeAfterLocal"`
-
-	RealtimeDuration          float64 `json:"realtimeDuration"`
-	RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"`
-	RealtimeSinceBoot         float64 `json:"realtimeSinceBoot"`
-	//Logs []string      `json: "logs"`
-
-	GitDir          string `json:"gitDir"`
-	GitRealDir      string `json:"gitRealDir"`
-	GitOriginRemote string `json:"gitOriginRemote"`
-	MachineId       string `json:"machineId"`
-
-	OsReleaseId         string `json:"osReleaseId"`
-	OsReleaseVersionId  string `json:"osReleaseVersionId"`
-	OsReleaseIdLike     string `json:"osReleaseIdLike"`
-	OsReleaseName       string `json:"osReleaseName"`
-	OsReleasePrettyName string `json:"osReleasePrettyName"`
-
-	ReshUuid     string `json:"reshUuid"`
-	ReshVersion  string `json:"reshVersion"`
-	ReshRevision string `json:"reshRevision"`
-}
-
-// ConvertRecord from FallbackRecord to Record
-func ConvertRecord(r *FallbackRecord) Record {
-	return Record{
-		// core
-		CmdLine:   r.CmdLine,
-		ExitCode:  r.ExitCode,
-		Shell:     r.Shell,
-		Uname:     r.Uname,
-		SessionId: r.SessionId,
-
-		// posix
-		// these two lines are the only reason we are doing this
-		Cols:  strconv.Itoa(r.Cols),
-		Lines: strconv.Itoa(r.Lines),
-
-		Home:  r.Home,
-		Lang:  r.Lang,
-		LcAll: r.LcAll,
-		Login: r.Login,
-		// Path:     r.path,
-		Pwd:      r.Pwd,
-		PwdAfter: r.PwdAfter,
-		ShellEnv: r.ShellEnv,
-		Term:     r.Term,
-
-		// non-posix
-		RealPwd:      r.RealPwd,
-		RealPwdAfter: r.RealPwdAfter,
-		Pid:          r.Pid,
-		SessionPid:   r.SessionPid,
-		Host:         r.Host,
-		Hosttype:     r.Hosttype,
-		Ostype:       r.Ostype,
-		Machtype:     r.Machtype,
-		Shlvl:        r.Shlvl,
-
-		// before after
-		TimezoneBefore: r.TimezoneBefore,
-		TimezoneAfter:  r.TimezoneAfter,
-
-		RealtimeBefore:      r.RealtimeBefore,
-		RealtimeAfter:       r.RealtimeAfter,
-		RealtimeBeforeLocal: r.RealtimeBeforeLocal,
-		RealtimeAfterLocal:  r.RealtimeAfterLocal,
-
-		RealtimeDuration:          r.RealtimeDuration,
-		RealtimeSinceSessionStart: r.RealtimeSinceSessionStart,
-		RealtimeSinceBoot:         r.RealtimeSinceBoot,
-
-		GitDir:          r.GitDir,
-		GitRealDir:      r.GitRealDir,
-		GitOriginRemote: r.GitOriginRemote,
-		MachineId:       r.MachineId,
-
-		OsReleaseId:         r.OsReleaseId,
-		OsReleaseVersionId:  r.OsReleaseVersionId,
-		OsReleaseIdLike:     r.OsReleaseIdLike,
-		OsReleaseName:       r.OsReleaseName,
-		OsReleasePrettyName: r.OsReleasePrettyName,
-
-		ReshUuid:     r.ReshUuid,
-		ReshVersion:  r.ReshVersion,
-		ReshRevision: r.ReshRevision,
-	}
-}
-
-// Enrich - adds additional fields to the record
-func (r *Record) Enrich() {
-	// Get command/first word from commandline
-	r.FirstWord = GetCommandFromCommandLine(r.CmdLine)
-	err := r.Validate()
-	if err != nil {
-		log.Println("Invalid command:", r.CmdLine)
-		r.Invalid = true
-	}
-	r.Invalid = false
-	// TODO: Detect and mark simple commands r.Simple
-}
-
-// Validate - returns error if the record is invalid
-func (r *Record) Validate() error {
-	return nil
-}
-
-// GetCommandFromCommandLine func
-func GetCommandFromCommandLine(cmdLine string) string {
-	args, err := shellwords.Parse(cmdLine)
-	if err != nil {
-		log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
-		return "<error>"
-	}
-	if len(args) > 0 {
-		return args[0]
-	}
-	return ""
-}
-
-// Config struct
-type Config struct {
-	Port int
-}
--- a/conf/config.toml
+++ b/conf/config.toml
--- a/data/sanitizer/copyright_information.md
+++ b/data/sanitizer/copyright_information.md
--- a/data/sanitizer/whitelist.txt
+++ b/data/sanitizer/whitelist.txt
--- a/evaluate/resh-evaluate.go
+++ b/evaluate/resh-evaluate.go
@ -1,340 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"os/exec"
-	"os/user"
-	"path/filepath"
-	"sort"
-
-	"github.com/curusarn/resh/common"
-)
-
-// Version from git set during build
-var Version string
-
-// Revision from git set during build
-var Revision string
-
-func main() {
-	usr, _ := user.Current()
-	dir := usr.HomeDir
-	historyPath := filepath.Join(dir, ".resh_history.json")
-	historyPathBatchMode := filepath.Join(dir, "resh_history.json")
-	sanitizedHistoryPath := filepath.Join(dir, "resh_history_sanitized.json")
-	// tmpPath := "/tmp/resh-evaluate-tmp.json"
-
-	showVersion := flag.Bool("version", false, "Show version and exit")
-	showRevision := flag.Bool("revision", false, "Show git revision and exit")
-	input := flag.String("input", "",
-		"Input file (default: "+historyPath+"OR"+sanitizedHistoryPath+
-			" depending on --sanitized-input option)")
-	// outputDir := flag.String("output", "/tmp/resh-evaluate", "Output directory")
-	sanitizedInput := flag.Bool("sanitized-input", false,
-		"Handle input as sanitized (also changes default value for input argument)")
-	plottingScript := flag.String("plotting-script", "resh-evaluate-plot.py", "Script to use for plotting")
-	inputDataRoot := flag.String("input-data-root", "",
-		"Input data root, enables batch mode, looks for files matching --input option")
-
-	flag.Parse()
-
-	// handle show{Version,Revision} options
-	if *showVersion == true {
-		fmt.Println(Version)
-		os.Exit(0)
-	}
-	if *showRevision == true {
-		fmt.Println(Revision)
-		os.Exit(0)
-	}
-
-	// handle batch mode
-	batchMode := false
-	if *inputDataRoot != "" {
-		batchMode = true
-	}
-	// set default input
-	if *input == "" {
-		if *sanitizedInput {
-			*input = sanitizedHistoryPath
-		} else if batchMode {
-			*input = historyPathBatchMode
-		} else {
-			*input = historyPath
-		}
-	}
-
-	evaluator := evaluator{sanitizedInput: *sanitizedInput, maxCandidates: 50, BatchMode: batchMode}
-	if batchMode {
-		err := evaluator.initBatchMode(*input, *inputDataRoot)
-		if err != nil {
-			log.Fatal("Evaluator initBatchMode() error:", err)
-		}
-	} else {
-		err := evaluator.init(*input)
-		if err != nil {
-			log.Fatal("Evaluator init() error:", err)
-		}
-	}
-
-	var strategies []strategy
-
-	// dummy := strategyDummy{}
-	// strategies = append(strategies, &dummy)
-
-	recent := strategyRecent{}
-	frequent := strategyFrequent{}
-	frequent.init()
-	directory := strategyDirectorySensitive{}
-	directory.init()
-
-	strategies = append(strategies, &recent, &frequent, &directory)
-
-	for _, strat := range strategies {
-		err := evaluator.evaluate(strat)
-		if err != nil {
-			log.Println("Evaluator evaluate() error:", err)
-		}
-	}
-
-	evaluator.calculateStatsAndPlot(*plottingScript)
-}
-
-type strategy interface {
-	GetTitleAndDescription() (string, string)
-	GetCandidates() []string
-	AddHistoryRecord(record *common.Record) error
-	ResetHistory() error
-}
-
-type matchJSON struct {
-	Match         bool
-	Distance      int
-	CharsRecalled int
-}
-
-type strategyJSON struct {
-	Title       string
-	Description string
-	Matches     []matchJSON
-}
-
-type deviceRecords struct {
-	Name    string
-	Records []common.Record
-}
-
-type userRecords struct {
-	Name    string
-	Devices []deviceRecords
-}
-
-type evaluator struct {
-	sanitizedInput bool
-	BatchMode      bool
-	maxCandidates  int
-	UsersRecords   []userRecords
-	Strategies     []strategyJSON
-}
-
-func (e *evaluator) initBatchMode(input string, inputDataRoot string) error {
-	e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot)
-	e.processRecords()
-	return nil
-}
-
-func (e *evaluator) init(inputPath string) error {
-	records := e.loadHistoryRecords(inputPath)
-	device := deviceRecords{Records: records}
-	user := userRecords{}
-	user.Devices = append(user.Devices, device)
-	e.UsersRecords = append(e.UsersRecords, user)
-	e.processRecords()
-	return nil
-}
-
-func (e *evaluator) calculateStatsAndPlot(scriptName string) {
-	evalJSON, err := json.Marshal(e)
-	if err != nil {
-		log.Fatal("json marshal error", err)
-	}
-	buffer := bytes.Buffer{}
-	buffer.Write(evalJSON)
-	// run python script to stat and plot/
-	cmd := exec.Command(scriptName)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.Stdin = &buffer
-	err = cmd.Run()
-	if err != nil {
-		log.Printf("Command finished with error: %v", err)
-	}
-}
-
-// enrich records and add them to serializable structure
-func (e *evaluator) processRecords() {
-	for i := range e.UsersRecords {
-		for j, device := range e.UsersRecords[i].Devices {
-			sessionIDs := map[string]uint64{}
-			var nextID uint64
-			nextID = 0
-			for k, record := range e.UsersRecords[i].Devices[j].Records {
-				id, found := sessionIDs[record.SessionId]
-				if found == false {
-					id = nextID
-					sessionIDs[record.SessionId] = id
-					nextID++
-				}
-				record.SeqSessionID = id
-				// assert
-				if record.Sanitized != e.sanitizedInput {
-					if e.sanitizedInput {
-						log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized")
-					}
-					log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present")
-				}
-
-				e.UsersRecords[i].Devices[j].Records[k].Enrich()
-				// device.Records = append(device.Records, record)
-			}
-			sort.SliceStable(e.UsersRecords[i].Devices[j].Records, func(x, y int) bool {
-				if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
-					return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal
-				}
-				return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID
-			})
-		}
-	}
-}
-
-func (e *evaluator) evaluate(strategy strategy) error {
-	title, description := strategy.GetTitleAndDescription()
-	strategyData := strategyJSON{Title: title, Description: description}
-	for _, record := range e.UsersRecords[0].Devices[0].Records {
-		candidates := strategy.GetCandidates()
-
-		matchFound := false
-		for i, candidate := range candidates {
-			// make an option (--calculate-total) to turn this on/off ?
-			// if i >= e.maxCandidates {
-			// 	break
-			// }
-			if candidate == record.CmdLine {
-				match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength}
-				strategyData.Matches = append(strategyData.Matches, match)
-				matchFound = true
-				break
-			}
-		}
-		if matchFound == false {
-			strategyData.Matches = append(strategyData.Matches, matchJSON{})
-		}
-		err := strategy.AddHistoryRecord(&record)
-		if err != nil {
-			log.Println("Error while evauating", err)
-			return err
-		}
-	}
-	e.Strategies = append(e.Strategies, strategyData)
-	return nil
-}
-
-func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords {
-	var records []userRecords
-	info, err := os.Stat(dataRootPath)
-	if err != nil {
-		log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")")
-	}
-	if info.IsDir() == false {
-		log.Fatal("Error:", dataRootPath, "is not a directory - exiting!")
-	}
-	users, err := ioutil.ReadDir(dataRootPath)
-	if err != nil {
-		log.Fatal("Could not read directory:", dataRootPath)
-	}
-	fmt.Println("Listing users in <", dataRootPath, ">...")
-	for _, user := range users {
-		userRecords := userRecords{Name: user.Name()}
-		userFullPath := filepath.Join(dataRootPath, user.Name())
-		if user.IsDir() == false {
-			log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.")
-			continue
-		}
-		fmt.Println()
-		fmt.Printf("*- %s\n", user.Name())
-		devices, err := ioutil.ReadDir(userFullPath)
-		if err != nil {
-			log.Fatal("Could not read directory:", userFullPath)
-		}
-		for _, device := range devices {
-			deviceRecords := deviceRecords{Name: device.Name()}
-			deviceFullPath := filepath.Join(userFullPath, device.Name())
-			if device.IsDir() == false {
-				log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.")
-				continue
-			}
-			fmt.Printf("   \\- %s\n", device.Name())
-			files, err := ioutil.ReadDir(deviceFullPath)
-			if err != nil {
-				log.Fatal("Could not read directory:", deviceFullPath)
-			}
-			for _, file := range files {
-				fileFullPath := filepath.Join(deviceFullPath, file.Name())
-				if file.Name() == fname {
-					fmt.Printf("      \\- %s - loading ...", file.Name())
-					// load the data
-					deviceRecords.Records = e.loadHistoryRecords(fileFullPath)
-					fmt.Println(" OK ✓")
-				} else {
-					fmt.Printf("      \\- %s - skipped\n", file.Name())
-				}
-			}
-			userRecords.Devices = append(userRecords.Devices, deviceRecords)
-		}
-		records = append(records, userRecords)
-	}
-	return records
-}
-
-func (e *evaluator) loadHistoryRecords(fname string) []common.Record {
-	file, err := os.Open(fname)
-	if err != nil {
-		log.Fatal("Open() resh history file error:", err)
-	}
-	defer file.Close()
-
-	var records []common.Record
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		record := common.Record{}
-		fallbackRecord := common.FallbackRecord{}
-		line := scanner.Text()
-		err = json.Unmarshal([]byte(line), &record)
-		if err != nil {
-			err = json.Unmarshal([]byte(line), &fallbackRecord)
-			if err != nil {
-				log.Println("Line:", line)
-				log.Fatal("Decoding error:", err)
-			}
-			record = common.ConvertRecord(&fallbackRecord)
-		}
-		if e.sanitizedInput == false {
-			if record.CmdLength != 0 {
-				log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?")
-			}
-			record.CmdLength = len(record.CmdLine)
-		}
-		if record.CmdLength == 0 {
-			log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.")
-		}
-		records = append(records, record)
-	}
-	return records
-}
--- a/evaluate/strategy-dummy.go
+++ b/evaluate/strategy-dummy.go
@ -1,24 +0,0 @@
-package main
-
-import "github.com/curusarn/resh/common"
-
-type strategyDummy struct {
-	history []string
-}
-
-func (s *strategyDummy) GetTitleAndDescription() (string, string) {
-	return "dummy", "Return empty candidate list"
-}
-
-func (s *strategyDummy) GetCandidates() []string {
-	return nil
-}
-
-func (s *strategyDummy) AddHistoryRecord(record *common.Record) error {
-	s.history = append(s.history, record.CmdLine)
-	return nil
-}
-
-func (s *strategyDummy) ResetHistory() error {
-	return nil
-}
--- a/evaluate/strategy-recent.go
+++ b/evaluate/strategy-recent.go
@ -1,32 +0,0 @@
-package main
-
-import "github.com/curusarn/resh/common"
-
-type strategyRecent struct {
-	history []string
-}
-
-func (s *strategyRecent) GetTitleAndDescription() (string, string) {
-	return "recent", "Use recent commands"
-}
-
-func (s *strategyRecent) GetCandidates() []string {
-	return s.history
-}
-
-func (s *strategyRecent) AddHistoryRecord(record *common.Record) error {
-	// remove previous occurance of record
-	for i, cmd := range s.history {
-		if cmd == record.CmdLine {
-			s.history = append(s.history[:i], s.history[i+1:]...)
-		}
-	}
-	// append new record
-	s.history = append([]string{record.CmdLine}, s.history...)
-	return nil
-}
-
-func (s *strategyRecent) ResetHistory() error {
-	s.history = nil
-	return nil
-}
--- a/go.mod
+++ b/go.mod
@ -5,7 +5,10 @@ go 1.12
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629
 	github.com/mattn/go-shellwords v1.0.6
+	github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7
+	github.com/schollz/progressbar v1.0.0
 	github.com/wcharczuk/go-chart v2.0.1+incompatible
 	github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa
 	golang.org/x/image v0.0.0-20190902063713-cb417be4ba39 // indirect
--- a/go.sum
+++ b/go.sum
@ -2,8 +2,14 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 h1:1dSBUfGlorLAua2CRx0zFN7kQsTpE2DQSmr7rrTNgY8=
+github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629/go.mod h1:mb5nS4uRANwOJSZj8rlCWAfAcGi72GGMIXx+xGOjA7M=
 github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7 h1:VsJjhYhufMGXICLwLYr8mFVMp8/A+YqmagMHnG/BA/4=
+github.com/mb-14/gomarkov v0.0.0-20190125094512-044dd0dcb5e7/go.mod h1:zQmHoMvvVJb7cxyt1wGT77lqUaeOFXlogOppOr4uHVo=
+github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM=
+github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs=
 github.com/wcharczuk/go-chart v2.0.1+incompatible h1:0pz39ZAycJFF7ju/1mepnk26RLVLBCWz1STcD3doU0A=
 github.com/wcharczuk/go-chart v2.0.1+incompatible/go.mod h1:PF5tmL4EIx/7Wf+hEkpCqYi5He4u90sw+0+6FhrryuE=
 github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa h1:rW+Lu6281ed/4XGuVIa4/YebTRNvoUJlfJ44ktEVwZk=
--- a/pkg/cfg/cfg.go
+++ b/pkg/cfg/cfg.go
@ -0,0 +1,6 @@
+package cfg
+
+// Config struct
+type Config struct {
+	Port int
+}
--- a/pkg/histanal/histeval.go
+++ b/pkg/histanal/histeval.go
@ -0,0 +1,246 @@
+package histanal
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"os/exec"
+
+	"github.com/curusarn/resh/pkg/records"
+	"github.com/curusarn/resh/pkg/strat"
+	"github.com/jpillora/longestcommon"
+
+	"github.com/schollz/progressbar"
+)
+
+type matchJSON struct {
+	Match         bool
+	Distance      int
+	CharsRecalled int
+}
+
+type multiMatchItemJSON struct {
+	Distance      int
+	CharsRecalled int
+}
+
+type multiMatchJSON struct {
+	Match   bool
+	Entries []multiMatchItemJSON
+}
+
+type strategyJSON struct {
+	Title         string
+	Description   string
+	Matches       []matchJSON
+	PrefixMatches []multiMatchJSON
+}
+
+// HistEval evaluates history
+type HistEval struct {
+	HistLoad
+	BatchMode     bool
+	maxCandidates int
+	Strategies    []strategyJSON
+}
+
+// NewHistEval constructs new HistEval
+func NewHistEval(inputPath string,
+	maxCandidates int, skipFailedCmds bool,
+	debugRecords float64, sanitizedInput bool) HistEval {
+
+	e := HistEval{
+		HistLoad: HistLoad{
+			skipFailedCmds: skipFailedCmds,
+			debugRecords:   debugRecords,
+			sanitizedInput: sanitizedInput,
+		},
+		maxCandidates: maxCandidates,
+		BatchMode:     false,
+	}
+	records := e.loadHistoryRecords(inputPath)
+	device := deviceRecords{Records: records}
+	user := userRecords{}
+	user.Devices = append(user.Devices, device)
+	e.UsersRecords = append(e.UsersRecords, user)
+	e.preprocessRecords()
+	return e
+}
+
+// NewHistEvalBatchMode constructs new HistEval in batch mode
+func NewHistEvalBatchMode(input string, inputDataRoot string,
+	maxCandidates int, skipFailedCmds bool,
+	debugRecords float64, sanitizedInput bool) HistEval {
+
+	e := HistEval{
+		HistLoad: HistLoad{
+			skipFailedCmds: skipFailedCmds,
+			debugRecords:   debugRecords,
+			sanitizedInput: sanitizedInput,
+		},
+		maxCandidates: maxCandidates,
+		BatchMode:     false,
+	}
+	e.UsersRecords = e.loadHistoryRecordsBatchMode(input, inputDataRoot)
+	e.preprocessRecords()
+	return e
+}
+
+func (e *HistEval) preprocessDeviceRecords(device deviceRecords) deviceRecords {
+	sessionIDs := map[string]uint64{}
+	var nextID uint64
+	nextID = 1 // start with 1 because 0 won't get saved to json
+	for k, record := range device.Records {
+		id, found := sessionIDs[record.SessionID]
+		if found == false {
+			id = nextID
+			sessionIDs[record.SessionID] = id
+			nextID++
+		}
+		device.Records[k].SeqSessionID = id
+		// assert
+		if record.Sanitized != e.sanitizedInput {
+			if e.sanitizedInput {
+				log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized")
+			}
+			log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present")
+		}
+		device.Records[k].SeqSessionID = id
+		if e.debugRecords > 0 && rand.Float64() < e.debugRecords {
+			device.Records[k].DebugThisRecord = true
+		}
+	}
+	// sort.SliceStable(device.Records, func(x, y int) bool {
+	// 	if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
+	// 		return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal
+	// 	}
+	// 	return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID
+	// })
+
+	// iterate from back and mark last record of each session
+	sessionIDSet := map[string]bool{}
+	for i := len(device.Records) - 1; i >= 0; i-- {
+		var record *records.EnrichedRecord
+		record = &device.Records[i]
+		if sessionIDSet[record.SessionID] {
+			continue
+		}
+		sessionIDSet[record.SessionID] = true
+		record.LastRecordOfSession = true
+	}
+	return device
+}
+
+// enrich records and add sequential session ID
+func (e *HistEval) preprocessRecords() {
+	for i := range e.UsersRecords {
+		for j := range e.UsersRecords[i].Devices {
+			e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j])
+		}
+	}
+}
+
+// Evaluate a given strategy
+func (e *HistEval) Evaluate(strategy strat.IStrategy) error {
+	title, description := strategy.GetTitleAndDescription()
+	log.Println("Evaluating strategy:", title, "-", description)
+	strategyData := strategyJSON{Title: title, Description: description}
+	for i := range e.UsersRecords {
+		for j := range e.UsersRecords[i].Devices {
+			bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records))
+			var prevRecord records.EnrichedRecord
+			for _, record := range e.UsersRecords[i].Devices[j].Records {
+				if e.skipFailedCmds && record.ExitCode != 0 {
+					continue
+				}
+				candidates := strategy.GetCandidates(records.Stripped(record))
+				if record.DebugThisRecord {
+					log.Println()
+					log.Println("===================================================")
+					log.Println("STRATEGY:", title, "-", description)
+					log.Println("===================================================")
+					log.Println("Previous record:")
+					if prevRecord.RealtimeBefore == 0 {
+						log.Println("== NIL")
+					} else {
+						rec, _ := prevRecord.ToString()
+						log.Println(rec)
+					}
+					log.Println("---------------------------------------------------")
+					log.Println("Recommendations for:")
+					rec, _ := record.ToString()
+					log.Println(rec)
+					log.Println("---------------------------------------------------")
+					for i, candidate := range candidates {
+						if i > 10 {
+							break
+						}
+						log.Println(string(candidate))
+					}
+					log.Println("===================================================")
+				}
+
+				matchFound := false
+				longestPrefixMatchLength := 0
+				multiMatch := multiMatchJSON{}
+				for i, candidate := range candidates {
+					// make an option (--calculate-total) to turn this on/off ?
+					// if i >= e.maxCandidates {
+					// 	break
+					// }
+					commonPrefixLength := len(longestcommon.Prefix([]string{candidate, record.CmdLine}))
+					if commonPrefixLength > longestPrefixMatchLength {
+						longestPrefixMatchLength = commonPrefixLength
+						prefixMatch := multiMatchItemJSON{Distance: i + 1, CharsRecalled: commonPrefixLength}
+						multiMatch.Match = true
+						multiMatch.Entries = append(multiMatch.Entries, prefixMatch)
+					}
+					if candidate == record.CmdLine {
+						match := matchJSON{Match: true, Distance: i + 1, CharsRecalled: record.CmdLength}
+						matchFound = true
+						strategyData.Matches = append(strategyData.Matches, match)
+						strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch)
+						break
+					}
+				}
+				if matchFound == false {
+					strategyData.Matches = append(strategyData.Matches, matchJSON{})
+					strategyData.PrefixMatches = append(strategyData.PrefixMatches, multiMatch)
+				}
+				err := strategy.AddHistoryRecord(&record)
+				if err != nil {
+					log.Println("Error while evauating", err)
+					return err
+				}
+				bar.Add(1)
+				prevRecord = record
+			}
+			strategy.ResetHistory()
+			fmt.Println()
+		}
+	}
+	e.Strategies = append(e.Strategies, strategyData)
+	return nil
+}
+
+// CalculateStatsAndPlot results
+func (e *HistEval) CalculateStatsAndPlot(scriptName string) {
+	evalJSON, err := json.Marshal(e)
+	if err != nil {
+		log.Fatal("json marshal error", err)
+	}
+	buffer := bytes.Buffer{}
+	buffer.Write(evalJSON)
+	// run python script to stat and plot/
+	cmd := exec.Command(scriptName)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = &buffer
+	err = cmd.Run()
+	if err != nil {
+		log.Printf("Command finished with error: %v", err)
+	}
+}
--- a/pkg/histanal/histload.go
+++ b/pkg/histanal/histload.go
@ -0,0 +1,179 @@
+package histanal
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+
+	"github.com/curusarn/resh/pkg/records"
+)
+
+type deviceRecords struct {
+	Name    string
+	Records []records.EnrichedRecord
+}
+
+type userRecords struct {
+	Name    string
+	Devices []deviceRecords
+}
+
+// HistLoad loads history
+type HistLoad struct {
+	UsersRecords   []userRecords
+	skipFailedCmds bool
+	sanitizedInput bool
+	debugRecords   float64
+}
+
+func (e *HistLoad) preprocessDeviceRecords(device deviceRecords) deviceRecords {
+	sessionIDs := map[string]uint64{}
+	var nextID uint64
+	nextID = 1 // start with 1 because 0 won't get saved to json
+	for k, record := range device.Records {
+		id, found := sessionIDs[record.SessionID]
+		if found == false {
+			id = nextID
+			sessionIDs[record.SessionID] = id
+			nextID++
+		}
+		device.Records[k].SeqSessionID = id
+		// assert
+		if record.Sanitized != e.sanitizedInput {
+			if e.sanitizedInput {
+				log.Fatal("ASSERT failed: '--sanitized-input' is present but data is not sanitized")
+			}
+			log.Fatal("ASSERT failed: data is sanitized but '--sanitized-input' is not present")
+		}
+		device.Records[k].SeqSessionID = id
+		if e.debugRecords > 0 && rand.Float64() < e.debugRecords {
+			device.Records[k].DebugThisRecord = true
+		}
+	}
+	// sort.SliceStable(device.Records, func(x, y int) bool {
+	// 	if device.Records[x].SeqSessionID == device.Records[y].SeqSessionID {
+	// 		return device.Records[x].RealtimeAfterLocal < device.Records[y].RealtimeAfterLocal
+	// 	}
+	// 	return device.Records[x].SeqSessionID < device.Records[y].SeqSessionID
+	// })
+
+	// iterate from back and mark last record of each session
+	sessionIDSet := map[string]bool{}
+	for i := len(device.Records) - 1; i >= 0; i-- {
+		var record *records.EnrichedRecord
+		record = &device.Records[i]
+		if sessionIDSet[record.SessionID] {
+			continue
+		}
+		sessionIDSet[record.SessionID] = true
+		record.LastRecordOfSession = true
+	}
+	return device
+}
+
+// enrich records and add sequential session ID
+func (e *HistLoad) preprocessRecords() {
+	for i := range e.UsersRecords {
+		for j := range e.UsersRecords[i].Devices {
+			e.UsersRecords[i].Devices[j] = e.preprocessDeviceRecords(e.UsersRecords[i].Devices[j])
+		}
+	}
+}
+
+func (e *HistLoad) loadHistoryRecordsBatchMode(fname string, dataRootPath string) []userRecords {
+	var records []userRecords
+	info, err := os.Stat(dataRootPath)
+	if err != nil {
+		log.Fatal("Error: Directory", dataRootPath, "does not exist - exiting! (", err, ")")
+	}
+	if info.IsDir() == false {
+		log.Fatal("Error:", dataRootPath, "is not a directory - exiting!")
+	}
+	users, err := ioutil.ReadDir(dataRootPath)
+	if err != nil {
+		log.Fatal("Could not read directory:", dataRootPath)
+	}
+	fmt.Println("Listing users in <", dataRootPath, ">...")
+	for _, user := range users {
+		userRecords := userRecords{Name: user.Name()}
+		userFullPath := filepath.Join(dataRootPath, user.Name())
+		if user.IsDir() == false {
+			log.Println("Warn: Unexpected file (not a directory) <", userFullPath, "> - skipping.")
+			continue
+		}
+		fmt.Println()
+		fmt.Printf("*- %s\n", user.Name())
+		devices, err := ioutil.ReadDir(userFullPath)
+		if err != nil {
+			log.Fatal("Could not read directory:", userFullPath)
+		}
+		for _, device := range devices {
+			deviceRecords := deviceRecords{Name: device.Name()}
+			deviceFullPath := filepath.Join(userFullPath, device.Name())
+			if device.IsDir() == false {
+				log.Println("Warn: Unexpected file (not a directory) <", deviceFullPath, "> - skipping.")
+				continue
+			}
+			fmt.Printf("   \\- %s\n", device.Name())
+			files, err := ioutil.ReadDir(deviceFullPath)
+			if err != nil {
+				log.Fatal("Could not read directory:", deviceFullPath)
+			}
+			for _, file := range files {
+				fileFullPath := filepath.Join(deviceFullPath, file.Name())
+				if file.Name() == fname {
+					fmt.Printf("      \\- %s - loading ...", file.Name())
+					// load the data
+					deviceRecords.Records = e.loadHistoryRecords(fileFullPath)
+					fmt.Println(" OK ✓")
+				} else {
+					fmt.Printf("      \\- %s - skipped\n", file.Name())
+				}
+			}
+			userRecords.Devices = append(userRecords.Devices, deviceRecords)
+		}
+		records = append(records, userRecords)
+	}
+	return records
+}
+
+func (e *HistLoad) loadHistoryRecords(fname string) []records.EnrichedRecord {
+	file, err := os.Open(fname)
+	if err != nil {
+		log.Fatal("Open() resh history file error:", err)
+	}
+	defer file.Close()
+
+	var recs []records.EnrichedRecord
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		record := records.Record{}
+		fallbackRecord := records.FallbackRecord{}
+		line := scanner.Text()
+		err = json.Unmarshal([]byte(line), &record)
+		if err != nil {
+			err = json.Unmarshal([]byte(line), &fallbackRecord)
+			if err != nil {
+				log.Println("Line:", line)
+				log.Fatal("Decoding error:", err)
+			}
+			record = records.ConvertRecord(&fallbackRecord)
+		}
+		if e.sanitizedInput == false {
+			if record.CmdLength != 0 {
+				log.Fatal("Assert failed - 'cmdLength' is set in raw data. Maybe you want to use '--sanitized-input' option?")
+			}
+			record.CmdLength = len(record.CmdLine)
+		}
+		if record.CmdLength == 0 {
+			log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.")
+		}
+		recs = append(recs, records.Enriched(record))
+	}
+	return recs
+}
--- a/pkg/records/records.go
+++ b/pkg/records/records.go
@ -0,0 +1,391 @@
+package records
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+)
+
+// BaseRecord - common base for Record and FallbackRecord
+type BaseRecord struct {
+	// core
+	CmdLine   string `json:"cmdLine"`
+	ExitCode  int    `json:"exitCode"`
+	Shell     string `json:"shell"`
+	Uname     string `json:"uname"`
+	SessionID string `json:"sessionId"`
+
+	// posix
+	Home  string `json:"home"`
+	Lang  string `json:"lang"`
+	LcAll string `json:"lcAll"`
+	Login string `json:"login"`
+	//Path     string `json:"path"`
+	Pwd      string `json:"pwd"`
+	PwdAfter string `json:"pwdAfter"`
+	ShellEnv string `json:"shellEnv"`
+	Term     string `json:"term"`
+
+	// non-posix"`
+	RealPwd      string `json:"realPwd"`
+	RealPwdAfter string `json:"realPwdAfter"`
+	Pid          int    `json:"pid"`
+	SessionPid   int    `json:"sessionPid"`
+	Host         string `json:"host"`
+	Hosttype     string `json:"hosttype"`
+	Ostype       string `json:"ostype"`
+	Machtype     string `json:"machtype"`
+	Shlvl        int    `json:"shlvl"`
+
+	// before after
+	TimezoneBefore string `json:"timezoneBefore"`
+	TimezoneAfter  string `json:"timezoneAfter"`
+
+	RealtimeBefore      float64 `json:"realtimeBefore"`
+	RealtimeAfter       float64 `json:"realtimeAfter"`
+	RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"`
+	RealtimeAfterLocal  float64 `json:"realtimeAfterLocal"`
+
+	RealtimeDuration          float64 `json:"realtimeDuration"`
+	RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"`
+	RealtimeSinceBoot         float64 `json:"realtimeSinceBoot"`
+	//Logs []string      `json: "logs"`
+
+	GitDir          string `json:"gitDir"`
+	GitRealDir      string `json:"gitRealDir"`
+	GitOriginRemote string `json:"gitOriginRemote"`
+	MachineID       string `json:"machineId"`
+
+	OsReleaseID         string `json:"osReleaseId"`
+	OsReleaseVersionID  string `json:"osReleaseVersionId"`
+	OsReleaseIDLike     string `json:"osReleaseIdLike"`
+	OsReleaseName       string `json:"osReleaseName"`
+	OsReleasePrettyName string `json:"osReleasePrettyName"`
+
+	ReshUUID     string `json:"reshUuid"`
+	ReshVersion  string `json:"reshVersion"`
+	ReshRevision string `json:"reshRevision"`
+
+	// added by sanitizatizer
+	Sanitized bool `json:"sanitized,omitempty"`
+	CmdLength int  `json:"cmdLength,omitempty"`
+}
+
+// Record representing single executed command with its metadata
+type Record struct {
+	BaseRecord
+
+	Cols  string `json:"cols"`
+	Lines string `json:"lines"`
+}
+
+// EnrichedRecord - record enriched with additional data
+type EnrichedRecord struct {
+	Record
+
+	// enriching fields - added "later"
+	Command             string   `json:"command"`
+	FirstWord           string   `json:"firstWord"`
+	Invalid             bool     `json:"invalid"`
+	SeqSessionID        uint64   `json:"seqSessionId"`
+	LastRecordOfSession bool     `json:"lastRecordOfSession"`
+	DebugThisRecord     bool     `json:"debugThisRecord"`
+	Errors              []string `json:"errors"`
+	// SeqSessionID uint64 `json:"seqSessionId,omitempty"`
+}
+
+// FallbackRecord when record is too old and can't be parsed into regular Record
+type FallbackRecord struct {
+	BaseRecord
+	// older version of the record where cols and lines are int
+
+	Cols  int `json:"cols"`  // notice the int type
+	Lines int `json:"lines"` // notice the int type
+}
+
+// ConvertRecord from FallbackRecord to Record
+func ConvertRecord(r *FallbackRecord) Record {
+	return Record{
+		BaseRecord: r.BaseRecord,
+		// these two lines are the only reason we are doing this
+		Cols:  strconv.Itoa(r.Cols),
+		Lines: strconv.Itoa(r.Lines),
+	}
+}
+
+// ToString - returns record the json
+func (r EnrichedRecord) ToString() (string, error) {
+	jsonRec, err := json.Marshal(r)
+	if err != nil {
+		return "marshalling error", err
+	}
+	return string(jsonRec), nil
+}
+
+// Enriched - returnd enriched record
+func Enriched(r Record) EnrichedRecord {
+	record := EnrichedRecord{Record: r}
+	// Get command/first word from commandline
+	var err error
+	record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine)
+	if err != nil {
+		record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error())
+		rec, _ := record.ToString()
+		log.Println("Invalid command:", rec)
+		record.Invalid = true
+		return record
+	}
+	err = r.Validate()
+	if err != nil {
+		record.Errors = append(record.Errors, "Validate error:"+err.Error())
+		rec, _ := record.ToString()
+		log.Println("Invalid command:", rec)
+		record.Invalid = true
+	}
+	return record
+	// TODO: Detect and mark simple commands r.Simple
+}
+
+// Validate - returns error if the record is invalid
+func (r *Record) Validate() error {
+	if r.CmdLine == "" {
+		return errors.New("There is no CmdLine")
+	}
+	if r.RealtimeBefore == 0 || r.RealtimeAfter == 0 {
+		return errors.New("There is no Time")
+	}
+	if r.RealtimeBeforeLocal == 0 || r.RealtimeAfterLocal == 0 {
+		return errors.New("There is no Local Time")
+	}
+	if r.RealPwd == "" || r.RealPwdAfter == "" {
+		return errors.New("There is no Real Pwd")
+	}
+	if r.Pwd == "" || r.PwdAfter == "" {
+		return errors.New("There is no Pwd")
+	}
+
+	// TimezoneBefore
+	// TimezoneAfter
+
+	// RealtimeDuration
+	// RealtimeSinceSessionStart - TODO: add later
+	// RealtimeSinceBoot  - TODO: add later
+
+	// device extras
+	// Host
+	// Hosttype
+	// Ostype
+	// Machtype
+	// OsReleaseID
+	// OsReleaseVersionID
+	// OsReleaseIDLike
+	// OsReleaseName
+	// OsReleasePrettyName
+
+	// session extras
+	// Term
+	// Shlvl
+
+	// static info
+	// Lang
+	// LcAll
+
+	// meta
+	// ReshUUID
+	// ReshVersion
+	// ReshRevision
+
+	// added by sanitizatizer
+	// Sanitized
+	// CmdLength
+	return nil
+}
+
+// SetCmdLine sets cmdLine and related members
+func (r *EnrichedRecord) SetCmdLine(cmdLine string) {
+	r.CmdLine = cmdLine
+	r.CmdLength = len(cmdLine)
+	r.ExitCode = 0
+	var err error
+	r.Command, r.FirstWord, err = GetCommandAndFirstWord(cmdLine)
+	if err != nil {
+		r.Errors = append(r.Errors, "GetCommandAndFirstWord error:"+err.Error())
+		// log.Println("Invalid command:", r.CmdLine)
+		r.Invalid = true
+	}
+}
+
+// Stripped returns record stripped of all info that is not available during prediction
+func Stripped(r EnrichedRecord) EnrichedRecord {
+	// clear the cmd itself
+	r.SetCmdLine("")
+	// replace after info with before info
+	r.PwdAfter = r.Pwd
+	r.RealPwdAfter = r.RealPwd
+	r.TimezoneAfter = r.TimezoneBefore
+	r.RealtimeAfter = r.RealtimeBefore
+	r.RealtimeAfterLocal = r.RealtimeBeforeLocal
+	// clear some more stuff
+	r.RealtimeDuration = 0
+	r.LastRecordOfSession = false
+	return r
+}
+
+// GetCommandAndFirstWord func
+func GetCommandAndFirstWord(cmdLine string) (string, string, error) {
+	args, err := shellwords.Parse(cmdLine)
+	if err != nil {
+		log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
+		return "", "", err
+	}
+	if len(args) == 0 {
+		return "", "", nil
+	}
+	i := 0
+	for true {
+		// commands in shell sometimes look like this `variable=something command argument otherArgument --option`
+		//		to get the command we skip over tokens that contain '='
+		if strings.ContainsRune(args[i], '=') && len(args) > i+1 {
+			i++
+			continue
+		}
+		return args[i], args[0], nil
+	}
+	log.Fatal("GetCommandAndFirstWord error: this should not happen!")
+	return "ERROR", "ERROR", errors.New("this should not happen - contact developer ;)")
+}
+
+// DistParams is used to supply params to Enrichedrecords.DistanceTo()
+type DistParams struct {
+	ExitCode  float64
+	MachineID float64
+	SessionID float64
+	Login     float64
+	Shell     float64
+	Pwd       float64
+	RealPwd   float64
+	Git       float64
+	Time      float64
+}
+
+// DistanceTo another record
+func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 {
+	var dist float64
+	dist = 0
+
+	// lev distance or something? TODO later
+	// CmdLine
+
+	// exit code
+	if r.ExitCode != r2.ExitCode {
+		if r.ExitCode == 0 || r2.ExitCode == 0 {
+			// one success + one error -> 1
+			dist += 1 * p.ExitCode
+		} else {
+			// two different errors
+			dist += 0.5 * p.ExitCode
+		}
+	}
+
+	// machine/device
+	if r.MachineID != r2.MachineID {
+		dist += 1 * p.MachineID
+	}
+	// Uname
+
+	// session
+	if r.SessionID != r2.SessionID {
+		dist += 1 * p.SessionID
+	}
+	// Pid - add because of nested shells?
+	// SessionPid
+
+	// user
+	if r.Login != r2.Login {
+		dist += 1 * p.Login
+	}
+	// Home
+
+	// shell
+	if r.Shell != r2.Shell {
+		dist += 1 * p.Shell
+	}
+	// ShellEnv
+
+	// pwd
+	if r.Pwd != r2.Pwd {
+		// TODO: compare using hierarchy
+		// TODO: make more important
+		dist += 1 * p.Pwd
+	}
+	if r.RealPwd != r2.RealPwd {
+		// TODO: -||-
+		dist += 1 * p.RealPwd
+	}
+	// PwdAfter
+	// RealPwdAfter
+
+	// git
+	if r.GitDir != r2.GitDir {
+		dist += 1 * p.Git
+	}
+	if r.GitRealDir != r2.GitRealDir {
+		dist += 1 * p.Git
+	}
+	if r.GitOriginRemote != r2.GitOriginRemote {
+		dist += 1 * p.Git
+	}
+
+	// time
+	// this can actually get negative for differences of less than one second which is fine
+	// distance grows by 1 with every order
+	distTime := math.Log10(math.Abs(r.RealtimeBefore-r2.RealtimeBefore)) * p.Time
+	if math.IsNaN(distTime) == false && math.IsInf(distTime, 0) == false {
+		dist += distTime
+	}
+	// RealtimeBeforeLocal
+	// RealtimeAfter
+	// RealtimeAfterLocal
+
+	// TimezoneBefore
+	// TimezoneAfter
+
+	// RealtimeDuration
+	// RealtimeSinceSessionStart - TODO: add later
+	// RealtimeSinceBoot  - TODO: add later
+
+	// device extras
+	// Host
+	// Hosttype
+	// Ostype
+	// Machtype
+	// OsReleaseID
+	// OsReleaseVersionID
+	// OsReleaseIDLike
+	// OsReleaseName
+	// OsReleasePrettyName
+
+	// session extras
+	// Term
+	// Shlvl
+
+	// static info
+	// Lang
+	// LcAll
+
+	// meta
+	// ReshUUID
+	// ReshVersion
+	// ReshRevision
+
+	// added by sanitizatizer
+	// Sanitized
+	// CmdLength
+
+	return dist
+}
--- a/pkg/records/records_test.go
+++ b/pkg/records/records_test.go
@ -0,0 +1,152 @@
+package records
+
+import (
+	"bufio"
+	"encoding/json"
+	"log"
+	"os"
+	"testing"
+)
+
+func GetTestRecords() []Record {
+	file, err := os.Open("testdata/resh_history.json")
+	if err != nil {
+		log.Fatal("Open() resh history file error:", err)
+	}
+	defer file.Close()
+
+	var recs []Record
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		record := Record{}
+		line := scanner.Text()
+		err = json.Unmarshal([]byte(line), &record)
+		if err != nil {
+			log.Println("Line:", line)
+			log.Fatal("Decoding error:", err)
+		}
+		recs = append(recs, record)
+	}
+	return recs
+}
+
+func GetTestEnrichedRecords() []EnrichedRecord {
+	var recs []EnrichedRecord
+	for _, rec := range GetTestRecords() {
+		recs = append(recs, Enriched(rec))
+	}
+	return recs
+}
+
+func TestToString(t *testing.T) {
+	for _, rec := range GetTestEnrichedRecords() {
+		_, err := rec.ToString()
+		if err != nil {
+			t.Error("ToString() failed")
+		}
+	}
+}
+
+func TestEnriched(t *testing.T) {
+	record := Record{BaseRecord: BaseRecord{CmdLine: "cmd arg1 arg2"}}
+	enriched := Enriched(record)
+	if enriched.FirstWord != "cmd" || enriched.Command != "cmd" {
+		t.Error("Enriched() returned reocord w/ wrong Command OR FirstWord")
+	}
+}
+
+func TestValidate(t *testing.T) {
+	record := EnrichedRecord{}
+	if record.Validate() == nil {
+		t.Error("Validate() didn't return an error for invalid record")
+	}
+	record.CmdLine = "cmd arg"
+	record.FirstWord = "cmd"
+	record.Command = "cmd"
+	time := 1234.5678
+	record.RealtimeBefore = time
+	record.RealtimeAfter = time
+	record.RealtimeBeforeLocal = time
+	record.RealtimeAfterLocal = time
+	pwd := "/pwd"
+	record.Pwd = pwd
+	record.PwdAfter = pwd
+	record.RealPwd = pwd
+	record.RealPwdAfter = pwd
+	if record.Validate() != nil {
+		t.Error("Validate() returned an error for a valid record")
+	}
+}
+
+func TestSetCmdLine(t *testing.T) {
+	record := EnrichedRecord{}
+	cmdline := "cmd arg1 arg2"
+	record.SetCmdLine(cmdline)
+	if record.CmdLine != cmdline || record.Command != "cmd" || record.FirstWord != "cmd" {
+		t.Error()
+	}
+}
+
+func TestStripped(t *testing.T) {
+	for _, rec := range GetTestEnrichedRecords() {
+		stripped := Stripped(rec)
+
+		// there should be no cmdline
+		if stripped.CmdLine != "" ||
+			stripped.FirstWord != "" ||
+			stripped.Command != "" {
+			t.Error("Stripped() returned record w/ info about CmdLine, Command OR FirstWord")
+		}
+		//  *after* fields should be overwritten by *before* fields
+		if stripped.PwdAfter != stripped.Pwd ||
+			stripped.RealPwdAfter != stripped.RealPwd ||
+			stripped.TimezoneAfter != stripped.TimezoneBefore ||
+			stripped.RealtimeAfter != stripped.RealtimeBefore ||
+			stripped.RealtimeAfterLocal != stripped.RealtimeBeforeLocal {
+			t.Error("Stripped() returned record w/ different *after* and *before* values - *after* fields should be overwritten by *before* fields")
+		}
+		// there should be no information about duration and session end
+		if stripped.RealtimeDuration != 0 ||
+			stripped.LastRecordOfSession != false {
+			t.Error("Stripped() returned record with too much information")
+		}
+	}
+}
+
+func TestGetCommandAndFirstWord(t *testing.T) {
+	cmd, stWord, err := GetCommandAndFirstWord("cmd arg1 arg2")
+	if err != nil || cmd != "cmd" || stWord != "cmd" {
+		t.Error("GetCommandAndFirstWord() returned wrong Command OR FirstWord")
+	}
+}
+
+func TestDistanceTo(t *testing.T) {
+	paramsFull := DistParams{
+		ExitCode:  1,
+		MachineID: 1,
+		SessionID: 1,
+		Login:     1,
+		Shell:     1,
+		Pwd:       1,
+		RealPwd:   1,
+		Git:       1,
+		Time:      1,
+	}
+	paramsZero := DistParams{}
+	var prevRec EnrichedRecord
+	for _, rec := range GetTestEnrichedRecords() {
+		dist := rec.DistanceTo(rec, paramsFull)
+		if dist != 0 {
+			t.Error("DistanceTo() itself should be always 0")
+		}
+		dist = rec.DistanceTo(prevRec, paramsFull)
+		if dist == 0 {
+			t.Error("DistanceTo() between two test records shouldn't be 0")
+		}
+		dist = rec.DistanceTo(prevRec, paramsZero)
+		if dist != 0 {
+			t.Error("DistanceTo() should be 0 when DistParams is all zeros")
+		}
+		prevRec = rec
+	}
+}
--- a/pkg/records/testdata/resh_history.json
+++ b/pkg/records/testdata/resh_history.json
@ -0,0 +1,27 @@
+{"cmdLine":"ls","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"d5c0fe70-c80b-4715-87cb-f8d8d5b4c673","cols":"80","lines":"24","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14560,"sessionPid":14560,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1566762905.173595,"realtimeAfter":1566762905.1894295,"realtimeBeforeLocal":1566770105.173595,"realtimeAfterLocal":1566770105.1894295,"realtimeDuration":0.015834569931030273,"realtimeSinceSessionStart":1.7122540473937988,"realtimeSinceBoot":20766.542254047396,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"find . -name applications","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567420001.2531302,"realtimeAfter":1567420002.4311218,"realtimeBeforeLocal":1567427201.2531302,"realtimeAfterLocal":1567427202.4311218,"realtimeDuration":1.1779916286468506,"realtimeSinceSessionStart":957.4848053455353,"realtimeSinceBoot":2336.594805345535,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"desktop-file-validate curusarn.sync-clipboards.desktop ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"c5251955-3a64-4353-952e-08d62a898694","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/.local/share/applications","pwdAfter":"/home/simon/.local/share/applications","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/.local/share/applications","realPwdAfter":"/home/simon/.local/share/applications","pid":3109,"sessionPid":3109,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567421748.2965438,"realtimeAfter":1567421748.3068867,"realtimeBeforeLocal":1567428948.2965438,"realtimeAfterLocal":1567428948.3068867,"realtimeDuration":0.010342836380004883,"realtimeSinceSessionStart":2704.528218984604,"realtimeSinceBoot":4083.6382189846036,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"cat /tmp/extensions | grep '.'","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461416.6871984,"realtimeAfter":1567461416.7336714,"realtimeBeforeLocal":1567468616.6871984,"realtimeAfterLocal":1567468616.7336714,"realtimeDuration":0.046473026275634766,"realtimeSinceSessionStart":21.45597553253174,"realtimeSinceBoot":43752.03597553253,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"cd git/resh/","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461667.8806899,"realtimeAfter":1567461667.8949044,"realtimeBeforeLocal":1567468867.8806899,"realtimeAfterLocal":1567468867.8949044,"realtimeDuration":0.014214515686035156,"realtimeSinceSessionStart":272.64946699142456,"realtimeSinceBoot":44003.229466991426,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461707.6467602,"realtimeAfter":1567461707.7177293,"realtimeBeforeLocal":1567468907.6467602,"realtimeAfterLocal":1567468907.7177293,"realtimeDuration":0.0709691047668457,"realtimeSinceSessionStart":312.4155373573303,"realtimeSinceBoot":44042.99553735733,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"cat /tmp/extensions | grep '^\\.' | cut -f1 |tr '[:upper:]' '[:lower:]' ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461722.813049,"realtimeAfter":1567461722.8280325,"realtimeBeforeLocal":1567468922.813049,"realtimeAfterLocal":1567468922.8280325,"realtimeDuration":0.014983415603637695,"realtimeSinceSessionStart":327.581826210022,"realtimeSinceBoot":44058.161826210024,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"tig","exitCode":127,"shell":"bash","uname":"Linux","sessionId":"f044cdbf-fd51-4c37-8528-dcd98fc7b6d9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":6887,"sessionPid":6887,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567461906.3896828,"realtimeAfter":1567461906.4084594,"realtimeBeforeLocal":1567469106.3896828,"realtimeAfterLocal":1567469106.4084594,"realtimeDuration":0.018776655197143555,"realtimeSinceSessionStart":511.1584599018097,"realtimeSinceBoot":44241.73845990181,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"752acb916f2a"}
+{"cmdLine":"resh-sanitize-history | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a3318c80-3521-4b22-aa64-ea0f6c641410","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":14601,"sessionPid":14601,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567547116.2430356,"realtimeAfter":1567547116.7547352,"realtimeBeforeLocal":1567554316.2430356,"realtimeAfterLocal":1567554316.7547352,"realtimeDuration":0.5116996765136719,"realtimeSinceSessionStart":15.841878414154053,"realtimeSinceBoot":30527.201878414155,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"sudo pacman -S ansible","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609042.0166302,"realtimeAfter":1567609076.9726007,"realtimeBeforeLocal":1567616242.0166302,"realtimeAfterLocal":1567616276.9726007,"realtimeDuration":34.95597052574158,"realtimeSinceSessionStart":1617.0794131755829,"realtimeSinceBoot":6120.029413175583,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"vagrant up","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609090.7359188,"realtimeAfter":1567609098.3125577,"realtimeBeforeLocal":1567616290.7359188,"realtimeAfterLocal":1567616298.3125577,"realtimeDuration":7.57663893699646,"realtimeSinceSessionStart":1665.798701763153,"realtimeSinceBoot":6168.748701763153,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"sudo modprobe vboxnetflt","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"64154f2d-a4bc-4463-a690-520080b61ead","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/kristin","pwdAfter":"/home/simon/git/kristin","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/kristin","realPwdAfter":"/home/simon/git/kristin","pid":5663,"sessionPid":5663,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567609143.2847652,"realtimeAfter":1567609143.3116078,"realtimeBeforeLocal":1567616343.2847652,"realtimeAfterLocal":1567616343.3116078,"realtimeDuration":0.026842594146728516,"realtimeSinceSessionStart":1718.3475482463837,"realtimeSinceBoot":6221.2975482463835,"gitDir":"/home/simon/git/kristin","gitRealDir":"/home/simon/git/kristin","gitOriginRemote":"git@gitlab.com:sucvut/kristin.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"echo $RANDOM","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"8ddacadc-6e73-483c-b347-4e18df204466","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":31387,"sessionPid":31387,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567727039.6540458,"realtimeAfter":1567727039.6629689,"realtimeBeforeLocal":1567734239.6540458,"realtimeAfterLocal":1567734239.6629689,"realtimeDuration":0.008923053741455078,"realtimeSinceSessionStart":1470.7667458057404,"realtimeSinceBoot":18495.01674580574,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"make resh-evaluate ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567977478.9672194,"realtimeAfter":1567977479.5449634,"realtimeBeforeLocal":1567984678.9672194,"realtimeAfterLocal":1567984679.5449634,"realtimeDuration":0.5777440071105957,"realtimeSinceSessionStart":5738.577540636063,"realtimeSinceBoot":20980.42754063606,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"cat ~/.resh_history.json | grep \"./resh-eval\" | jq","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"105","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1567986105.3988302,"realtimeAfter":1567986105.4809113,"realtimeBeforeLocal":1567993305.3988302,"realtimeAfterLocal":1567993305.4809113,"realtimeDuration":0.08208107948303223,"realtimeSinceSessionStart":14365.00915145874,"realtimeSinceBoot":29606.85915145874,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"git c \"add sanitized flag to record, add Enrich() to record\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063976.9103937,"realtimeAfter":1568063976.9326868,"realtimeBeforeLocal":1568071176.9103937,"realtimeAfterLocal":1568071176.9326868,"realtimeDuration":0.0222930908203125,"realtimeSinceSessionStart":92236.52071499825,"realtimeSinceBoot":107478.37071499825,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"git s","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063978.2340608,"realtimeAfter":1568063978.252463,"realtimeBeforeLocal":1568071178.2340608,"realtimeAfterLocal":1568071178.252463,"realtimeDuration":0.0184023380279541,"realtimeSinceSessionStart":92237.84438204765,"realtimeSinceBoot":107479.69438204766,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"git a evaluate/results.go ","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568063989.0446353,"realtimeAfter":1568063989.2452207,"realtimeBeforeLocal":1568071189.0446353,"realtimeAfterLocal":1568071189.2452207,"realtimeDuration":0.20058536529541016,"realtimeSinceSessionStart":92248.65495657921,"realtimeSinceBoot":107490.50495657921,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"sudo pacman -S python-pip","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072068.3557143,"realtimeAfter":1568072070.7509863,"realtimeBeforeLocal":1568079268.3557143,"realtimeAfterLocal":1568079270.7509863,"realtimeDuration":2.3952720165252686,"realtimeSinceSessionStart":100327.96603560448,"realtimeSinceBoot":115569.81603560448,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"pip3 install matplotlib","exitCode":1,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072088.5575967,"realtimeAfter":1568072094.372314,"realtimeBeforeLocal":1568079288.5575967,"realtimeAfterLocal":1568079294.372314,"realtimeDuration":5.8147172927856445,"realtimeSinceSessionStart":100348.16791796684,"realtimeSinceBoot":115590.01791796685,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"sudo pip3 install matplotlib","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568072106.138616,"realtimeAfter":1568072115.1124601,"realtimeBeforeLocal":1568079306.138616,"realtimeAfterLocal":1568079315.1124601,"realtimeDuration":8.973844051361084,"realtimeSinceSessionStart":100365.7489373684,"realtimeSinceBoot":115607.5989373684,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"./resh-evaluate --plotting-script evaluate/resh-evaluate-plot.py --input ~/git/resh_private/history_data/simon/dell/resh_history.json ","exitCode":130,"shell":"bash","uname":"Linux","sessionId":"93998b68-ec48-4e48-9e4a-b37b39f5439e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":9463,"sessionPid":9463,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1568076266.9364285,"realtimeAfter":1568076288.1131275,"realtimeBeforeLocal":1568083466.9364285,"realtimeAfterLocal":1568083488.1131275,"realtimeDuration":21.176698923110962,"realtimeSinceSessionStart":104526.54674983025,"realtimeSinceBoot":119768.39674983025,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.1","reshRevision":"737bc0a4df38","cmdLength":0}
+{"cmdLine":"git c \"Add a bunch of useless comments to make linter happy\"","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"04050353-a97d-4435-9248-f47dd08b2f2a","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":14702,"sessionPid":14702,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569456045.8763022,"realtimeAfter":1569456045.9030173,"realtimeBeforeLocal":1569463245.8763022,"realtimeAfterLocal":1569463245.9030173,"realtimeDuration":0.02671504020690918,"realtimeSinceSessionStart":2289.789242744446,"realtimeSinceBoot":143217.91924274445,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false}
+{"cmdLine":"fuck","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"a4aadf03-610d-4731-ba94-5b7ce21e7bb9","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":3413,"sessionPid":3413,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569687682.4250975,"realtimeAfter":1569687682.5877323,"realtimeBeforeLocal":1569694882.4250975,"realtimeAfterLocal":1569694882.5877323,"realtimeDuration":0.16263484954833984,"realtimeSinceSessionStart":264603.49496507645,"realtimeSinceBoot":374854.48496507644,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false}
+{"cmdLine":"code .","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709366.523767,"realtimeAfter":1569709367.516908,"realtimeBeforeLocal":1569716566.523767,"realtimeAfterLocal":1569716567.516908,"realtimeDuration":0.9931409358978271,"realtimeSinceSessionStart":23846.908839941025,"realtimeSinceBoot":396539.888839941,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false}
+{"cmdLine":"make test","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"87c7ab14-ae51-408d-adbc-fc4f9d28de6e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon/git/resh","pwdAfter":"/home/simon/git/resh","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon/git/resh","realPwdAfter":"/home/simon/git/resh","pid":31947,"sessionPid":31947,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709371.89966,"realtimeAfter":1569709377.430194,"realtimeBeforeLocal":1569716571.89966,"realtimeAfterLocal":1569716577.430194,"realtimeDuration":5.530533790588379,"realtimeSinceSessionStart":23852.284733057022,"realtimeSinceBoot":396545.264733057,"gitDir":"/home/simon/git/resh","gitRealDir":"/home/simon/git/resh","gitOriginRemote":"git@github.com:curusarn/resh.git","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false}
+{"cmdLine":"mkdir ~/git/resh/testdata","exitCode":0,"shell":"bash","uname":"Linux","sessionId":"71529b60-2e7b-4d5b-8dc1-6d0740b58e9e","cols":"211","lines":"56","home":"/home/simon","lang":"en_US.UTF-8","lcAll":"","login":"simon","pwd":"/home/simon","pwdAfter":"/home/simon","shellEnv":"/bin/bash","term":"xterm-256color","realPwd":"/home/simon","realPwdAfter":"/home/simon","pid":21224,"sessionPid":21224,"host":"simon-pc","hosttype":"x86_64","ostype":"linux-gnu","machtype":"x86_64-pc-linux-gnu","shlvl":1,"timezoneBefore":"+0200","timezoneAfter":"+0200","realtimeBefore":1569709838.4642656,"realtimeAfter":1569709838.4718792,"realtimeBeforeLocal":1569717038.4642656,"realtimeAfterLocal":1569717038.4718792,"realtimeDuration":0.007613658905029297,"realtimeSinceSessionStart":9.437154054641724,"realtimeSinceBoot":397011.02715405467,"gitDir":"","gitRealDir":"","gitOriginRemote":"","machineId":"c70365240bc647f09e2490722cc8186b","osReleaseId":"manjaro","osReleaseVersionId":"","osReleaseIdLike":"arch","osReleaseName":"Manjaro Linux","osReleasePrettyName":"Manjaro Linux","reshUuid":"","reshVersion":"1.1.3","reshRevision":"188d8b420493","sanitized":false}
--- a/evaluate/strategy-directory-sensitive.go
+++ b/evaluate/strategy-directory-sensitive.go
@ -1,27 +1,30 @@
-package main
+package strat

-import (
-	"github.com/curusarn/resh/common"
-)
+import "github.com/curusarn/resh/pkg/records"

-type strategyDirectorySensitive struct {
+// DirectorySensitive prediction/recommendation strategy
+type DirectorySensitive struct {
 	history map[string][]string
 	lastPwd string
 }

-func (s *strategyDirectorySensitive) init() {
+// Init see name
+func (s *DirectorySensitive) Init() {
 	s.history = map[string][]string{}
 }

-func (s *strategyDirectorySensitive) GetTitleAndDescription() (string, string) {
+// GetTitleAndDescription see name
+func (s *DirectorySensitive) GetTitleAndDescription() (string, string) {
 	return "directory sensitive (recent)", "Use recent commands executed is the same directory"
 }

-func (s *strategyDirectorySensitive) GetCandidates() []string {
+// GetCandidates see name
+func (s *DirectorySensitive) GetCandidates() []string {
 	return s.history[s.lastPwd]
 }

-func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) error {
+// AddHistoryRecord see name
+func (s *DirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error {
 	// work on history for PWD
 	pwd := record.Pwd
 	// remove previous occurance of record
@ -36,7 +39,9 @@ func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.Record) err
 	return nil
 }

-func (s *strategyDirectorySensitive) ResetHistory() error {
+// ResetHistory see name
+func (s *DirectorySensitive) ResetHistory() error {
+	s.Init()
 	s.history = map[string][]string{}
 	return nil
 }
--- a/pkg/strat/dummy.go
+++ b/pkg/strat/dummy.go
@ -0,0 +1,29 @@
+package strat
+
+import "github.com/curusarn/resh/pkg/records"
+
+// Dummy prediction/recommendation strategy
+type Dummy struct {
+	history []string
+}
+
+// GetTitleAndDescription see name
+func (s *Dummy) GetTitleAndDescription() (string, string) {
+	return "dummy", "Return empty candidate list"
+}
+
+// GetCandidates see name
+func (s *Dummy) GetCandidates() []string {
+	return nil
+}
+
+// AddHistoryRecord see name
+func (s *Dummy) AddHistoryRecord(record *records.EnrichedRecord) error {
+	s.history = append(s.history, record.CmdLine)
+	return nil
+}
+
+// ResetHistory see name
+func (s *Dummy) ResetHistory() error {
+	return nil
+}
--- a/pkg/strat/dynamic-record-distance.go
+++ b/pkg/strat/dynamic-record-distance.go
@ -0,0 +1,91 @@
+package strat
+
+import (
+	"math"
+	"sort"
+	"strconv"
+
+	"github.com/curusarn/resh/pkg/records"
+)
+
+// DynamicRecordDistance prediction/recommendation strategy
+type DynamicRecordDistance struct {
+	history            []records.EnrichedRecord
+	DistParams         records.DistParams
+	pwdHistogram       map[string]int
+	realPwdHistogram   map[string]int
+	gitOriginHistogram map[string]int
+	MaxDepth           int
+	Label              string
+}
+
+type strDynDistEntry struct {
+	cmdLine  string
+	distance float64
+}
+
+// Init see name
+func (s *DynamicRecordDistance) Init() {
+	s.history = nil
+	s.pwdHistogram = map[string]int{}
+	s.realPwdHistogram = map[string]int{}
+	s.gitOriginHistogram = map[string]int{}
+}
+
+// GetTitleAndDescription see name
+func (s *DynamicRecordDistance) GetTitleAndDescription() (string, string) {
+	return "dynamic record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use TF-IDF record distance to recommend commands"
+}
+
+func (s *DynamicRecordDistance) idf(count int) float64 {
+	return math.Log(float64(len(s.history)) / float64(count))
+}
+
+// GetCandidates see name
+func (s *DynamicRecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string {
+	if len(s.history) == 0 {
+		return nil
+	}
+	var mapItems []strDynDistEntry
+	for i, record := range s.history {
+		if s.MaxDepth != 0 && i > s.MaxDepth {
+			break
+		}
+		distParams := records.DistParams{
+			Pwd:       s.DistParams.Pwd * s.idf(s.pwdHistogram[strippedRecord.PwdAfter]),
+			RealPwd:   s.DistParams.RealPwd * s.idf(s.realPwdHistogram[strippedRecord.RealPwdAfter]),
+			Git:       s.DistParams.Git * s.idf(s.gitOriginHistogram[strippedRecord.GitOriginRemote]),
+			Time:      s.DistParams.Time,
+			SessionID: s.DistParams.SessionID,
+		}
+		distance := record.DistanceTo(strippedRecord, distParams)
+		mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance})
+	}
+	sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance })
+	var hist []string
+	histSet := map[string]bool{}
+	for _, item := range mapItems {
+		if histSet[item.cmdLine] {
+			continue
+		}
+		histSet[item.cmdLine] = true
+		hist = append(hist, item.cmdLine)
+	}
+	return hist
+}
+
+// AddHistoryRecord see name
+func (s *DynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error {
+	// append record to front
+	s.history = append([]records.EnrichedRecord{*record}, s.history...)
+	s.pwdHistogram[record.Pwd]++
+	s.realPwdHistogram[record.RealPwd]++
+	s.gitOriginHistogram[record.GitOriginRemote]++
+	return nil
+}
+
+// ResetHistory see name
+func (s *DynamicRecordDistance) ResetHistory() error {
+	s.Init()
+	return nil
+}
--- a/evaluate/strategy-frequent.go
+++ b/evaluate/strategy-frequent.go
@ -1,12 +1,13 @@
-package main
+package strat

 import (
 	"sort"

-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 )

-type strategyFrequent struct {
+// Frequent prediction/recommendation strategy
+type Frequent struct {
 	history map[string]int
 }

@ -15,15 +16,18 @@ type strFrqEntry struct {
 	count   int
 }

-func (s *strategyFrequent) init() {
+// Init see name
+func (s *Frequent) Init() {
 	s.history = map[string]int{}
 }

-func (s *strategyFrequent) GetTitleAndDescription() (string, string) {
+// GetTitleAndDescription see name
+func (s *Frequent) GetTitleAndDescription() (string, string) {
 	return "frequent", "Use frequent commands"
 }

-func (s *strategyFrequent) GetCandidates() []string {
+// GetCandidates see name
+func (s *Frequent) GetCandidates() []string {
 	var mapItems []strFrqEntry
 	for cmdLine, count := range s.history {
 		mapItems = append(mapItems, strFrqEntry{cmdLine, count})
@ -36,12 +40,14 @@ func (s *strategyFrequent) GetCandidates() []string {
 	return hist
 }

-func (s *strategyFrequent) AddHistoryRecord(record *common.Record) error {
+// AddHistoryRecord see name
+func (s *Frequent) AddHistoryRecord(record *records.EnrichedRecord) error {
 	s.history[record.CmdLine]++
 	return nil
 }

-func (s *strategyFrequent) ResetHistory() error {
-	s.history = map[string]int{}
+// ResetHistory see name
+func (s *Frequent) ResetHistory() error {
+	s.Init()
 	return nil
 }
--- a/pkg/strat/markov-chain-cmd.go
+++ b/pkg/strat/markov-chain-cmd.go
@ -0,0 +1,97 @@
+package strat
+
+import (
+	"sort"
+	"strconv"
+
+	"github.com/curusarn/resh/pkg/records"
+	"github.com/mb-14/gomarkov"
+)
+
+// MarkovChainCmd prediction/recommendation strategy
+type MarkovChainCmd struct {
+	Order       int
+	history     []strMarkCmdHistoryEntry
+	historyCmds []string
+}
+
+type strMarkCmdHistoryEntry struct {
+	cmd     string
+	cmdLine string
+}
+
+type strMarkCmdEntry struct {
+	cmd       string
+	transProb float64
+}
+
+// Init see name
+func (s *MarkovChainCmd) Init() {
+	s.history = nil
+	s.historyCmds = nil
+}
+
+// GetTitleAndDescription see name
+func (s *MarkovChainCmd) GetTitleAndDescription() (string, string) {
+	return "command-based markov chain (order " + strconv.Itoa(s.Order) + ")", "Use command-based markov chain to recommend commands"
+}
+
+// GetCandidates see name
+func (s *MarkovChainCmd) GetCandidates() []string {
+	if len(s.history) < s.Order {
+		var hist []string
+		for _, item := range s.history {
+			hist = append(hist, item.cmdLine)
+		}
+		return hist
+	}
+	chain := gomarkov.NewChain(s.Order)
+
+	chain.Add(s.historyCmds)
+
+	cmdsSet := map[string]bool{}
+	var entries []strMarkCmdEntry
+	for _, cmd := range s.historyCmds {
+		if cmdsSet[cmd] {
+			continue
+		}
+		cmdsSet[cmd] = true
+		prob, _ := chain.TransitionProbability(cmd, s.historyCmds[len(s.historyCmds)-s.Order:])
+		entries = append(entries, strMarkCmdEntry{cmd: cmd, transProb: prob})
+	}
+	sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb })
+	var hist []string
+	histSet := map[string]bool{}
+	for i := len(s.history) - 1; i >= 0; i-- {
+		if histSet[s.history[i].cmdLine] {
+			continue
+		}
+		histSet[s.history[i].cmdLine] = true
+		if s.history[i].cmd == entries[0].cmd {
+			hist = append(hist, s.history[i].cmdLine)
+		}
+	}
+	// log.Println("################")
+	// log.Println(s.history[len(s.history)-s.order:])
+	// log.Println(" -> ")
+	// x := math.Min(float64(len(hist)), 3)
+	// log.Println(entries[:int(x)])
+	// x = math.Min(float64(len(hist)), 5)
+	// log.Println(hist[:int(x)])
+	// log.Println("################")
+	return hist
+}
+
+// AddHistoryRecord see name
+func (s *MarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error {
+	s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command})
+	s.historyCmds = append(s.historyCmds, record.Command)
+	// s.historySet[record.CmdLine] = true
+	return nil
+}
+
+// ResetHistory see name
+func (s *MarkovChainCmd) ResetHistory() error {
+	s.Init()
+	return nil
+}
--- a/pkg/strat/markov-chain.go
+++ b/pkg/strat/markov-chain.go
@ -0,0 +1,76 @@
+package strat
+
+import (
+	"sort"
+	"strconv"
+
+	"github.com/curusarn/resh/pkg/records"
+	"github.com/mb-14/gomarkov"
+)
+
+// MarkovChain prediction/recommendation strategy
+type MarkovChain struct {
+	Order   int
+	history []string
+}
+
+type strMarkEntry struct {
+	cmdLine   string
+	transProb float64
+}
+
+// Init see name
+func (s *MarkovChain) Init() {
+	s.history = nil
+}
+
+// GetTitleAndDescription see name
+func (s *MarkovChain) GetTitleAndDescription() (string, string) {
+	return "markov chain (order " + strconv.Itoa(s.Order) + ")", "Use markov chain to recommend commands"
+}
+
+// GetCandidates see name
+func (s *MarkovChain) GetCandidates() []string {
+	if len(s.history) < s.Order {
+		return s.history
+	}
+	chain := gomarkov.NewChain(s.Order)
+
+	chain.Add(s.history)
+
+	cmdLinesSet := map[string]bool{}
+	var entries []strMarkEntry
+	for _, cmdLine := range s.history {
+		if cmdLinesSet[cmdLine] {
+			continue
+		}
+		cmdLinesSet[cmdLine] = true
+		prob, _ := chain.TransitionProbability(cmdLine, s.history[len(s.history)-s.Order:])
+		entries = append(entries, strMarkEntry{cmdLine: cmdLine, transProb: prob})
+	}
+	sort.Slice(entries, func(i int, j int) bool { return entries[i].transProb > entries[j].transProb })
+	var hist []string
+	for _, item := range entries {
+		hist = append(hist, item.cmdLine)
+	}
+	// log.Println("################")
+	// log.Println(s.history[len(s.history)-s.order:])
+	// log.Println(" -> ")
+	// x := math.Min(float64(len(hist)), 5)
+	// log.Println(hist[:int(x)])
+	// log.Println("################")
+	return hist
+}
+
+// AddHistoryRecord see name
+func (s *MarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error {
+	s.history = append(s.history, record.CmdLine)
+	// s.historySet[record.CmdLine] = true
+	return nil
+}
+
+// ResetHistory see name
+func (s *MarkovChain) ResetHistory() error {
+	s.Init()
+	return nil
+}
--- a/pkg/strat/random.go
+++ b/pkg/strat/random.go
@ -0,0 +1,57 @@
+package strat
+
+import (
+	"math/rand"
+	"time"
+
+	"github.com/curusarn/resh/pkg/records"
+)
+
+// Random prediction/recommendation strategy
+type Random struct {
+	CandidatesSize int
+	history        []string
+	historySet     map[string]bool
+}
+
+// Init see name
+func (s *Random) Init() {
+	s.history = nil
+	s.historySet = map[string]bool{}
+}
+
+// GetTitleAndDescription see name
+func (s *Random) GetTitleAndDescription() (string, string) {
+	return "random", "Use random commands"
+}
+
+// GetCandidates see name
+func (s *Random) GetCandidates() []string {
+	seed := time.Now().UnixNano()
+	rand.Seed(seed)
+	var candidates []string
+	candidateSet := map[string]bool{}
+	for len(candidates) < s.CandidatesSize && len(candidates)*2 < len(s.historySet) {
+		x := rand.Intn(len(s.history))
+		candidate := s.history[x]
+		if candidateSet[candidate] == false {
+			candidateSet[candidate] = true
+			candidates = append(candidates, candidate)
+			continue
+		}
+	}
+	return candidates
+}
+
+// AddHistoryRecord see name
+func (s *Random) AddHistoryRecord(record *records.EnrichedRecord) error {
+	s.history = append([]string{record.CmdLine}, s.history...)
+	s.historySet[record.CmdLine] = true
+	return nil
+}
+
+// ResetHistory see name
+func (s *Random) ResetHistory() error {
+	s.Init()
+	return nil
+}
--- a/pkg/strat/recent-bash.go
+++ b/pkg/strat/recent-bash.go
@ -0,0 +1,56 @@
+package strat
+
+import "github.com/curusarn/resh/pkg/records"
+
+// RecentBash prediction/recommendation strategy
+type RecentBash struct {
+	histfile         []string
+	histfileSnapshot map[string][]string
+	history          map[string][]string
+}
+
+// Init see name
+func (s *RecentBash) Init() {
+	s.histfileSnapshot = map[string][]string{}
+	s.history = map[string][]string{}
+}
+
+// GetTitleAndDescription see name
+func (s *RecentBash) GetTitleAndDescription() (string, string) {
+	return "recent (bash-like)", "Behave like bash"
+}
+
+// GetCandidates see name
+func (s *RecentBash) GetCandidates(strippedRecord records.EnrichedRecord) []string {
+	// populate the local history from histfile
+	if s.histfileSnapshot[strippedRecord.SessionID] == nil {
+		s.histfileSnapshot[strippedRecord.SessionID] = s.histfile
+	}
+	return append(s.history[strippedRecord.SessionID], s.histfileSnapshot[strippedRecord.SessionID]...)
+}
+
+// AddHistoryRecord see name
+func (s *RecentBash) AddHistoryRecord(record *records.EnrichedRecord) error {
+	// remove previous occurance of record
+	for i, cmd := range s.history[record.SessionID] {
+		if cmd == record.CmdLine {
+			s.history[record.SessionID] = append(s.history[record.SessionID][:i], s.history[record.SessionID][i+1:]...)
+		}
+	}
+	// append new record
+	s.history[record.SessionID] = append([]string{record.CmdLine}, s.history[record.SessionID]...)
+
+	if record.LastRecordOfSession {
+		// append history of the session to histfile and clear session history
+		s.histfile = append(s.history[record.SessionID], s.histfile...)
+		s.histfileSnapshot[record.SessionID] = nil
+		s.history[record.SessionID] = nil
+	}
+	return nil
+}
+
+// ResetHistory see name
+func (s *RecentBash) ResetHistory() error {
+	s.Init()
+	return nil
+}
--- a/pkg/strat/recent.go
+++ b/pkg/strat/recent.go
@ -0,0 +1,37 @@
+package strat
+
+import "github.com/curusarn/resh/pkg/records"
+
+// Recent prediction/recommendation strategy
+type Recent struct {
+	history []string
+}
+
+// GetTitleAndDescription see name
+func (s *Recent) GetTitleAndDescription() (string, string) {
+	return "recent", "Use recent commands"
+}
+
+// GetCandidates see name
+func (s *Recent) GetCandidates() []string {
+	return s.history
+}
+
+// AddHistoryRecord see name
+func (s *Recent) AddHistoryRecord(record *records.EnrichedRecord) error {
+	// remove previous occurance of record
+	for i, cmd := range s.history {
+		if cmd == record.CmdLine {
+			s.history = append(s.history[:i], s.history[i+1:]...)
+		}
+	}
+	// append new record
+	s.history = append([]string{record.CmdLine}, s.history...)
+	return nil
+}
+
+// ResetHistory see name
+func (s *Recent) ResetHistory() error {
+	s.history = nil
+	return nil
+}
--- a/pkg/strat/record-distance.go
+++ b/pkg/strat/record-distance.go
@ -0,0 +1,70 @@
+package strat
+
+import (
+	"sort"
+	"strconv"
+
+	"github.com/curusarn/resh/pkg/records"
+)
+
+// RecordDistance prediction/recommendation strategy
+type RecordDistance struct {
+	history    []records.EnrichedRecord
+	DistParams records.DistParams
+	MaxDepth   int
+	Label      string
+}
+
+type strDistEntry struct {
+	cmdLine  string
+	distance float64
+}
+
+// Init see name
+func (s *RecordDistance) Init() {
+	s.history = nil
+}
+
+// GetTitleAndDescription see name
+func (s *RecordDistance) GetTitleAndDescription() (string, string) {
+	return "record distance (depth:" + strconv.Itoa(s.MaxDepth) + ";" + s.Label + ")", "Use record distance to recommend commands"
+}
+
+// GetCandidates see name
+func (s *RecordDistance) GetCandidates(strippedRecord records.EnrichedRecord) []string {
+	if len(s.history) == 0 {
+		return nil
+	}
+	var mapItems []strDistEntry
+	for i, record := range s.history {
+		if s.MaxDepth != 0 && i > s.MaxDepth {
+			break
+		}
+		distance := record.DistanceTo(strippedRecord, s.DistParams)
+		mapItems = append(mapItems, strDistEntry{record.CmdLine, distance})
+	}
+	sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance })
+	var hist []string
+	histSet := map[string]bool{}
+	for _, item := range mapItems {
+		if histSet[item.cmdLine] {
+			continue
+		}
+		histSet[item.cmdLine] = true
+		hist = append(hist, item.cmdLine)
+	}
+	return hist
+}
+
+// AddHistoryRecord see name
+func (s *RecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error {
+	// append record to front
+	s.history = append([]records.EnrichedRecord{*record}, s.history...)
+	return nil
+}
+
+// ResetHistory see name
+func (s *RecordDistance) ResetHistory() error {
+	s.Init()
+	return nil
+}
--- a/pkg/strat/strat.go
+++ b/pkg/strat/strat.go
@ -0,0 +1,46 @@
+package strat
+
+import (
+	"github.com/curusarn/resh/pkg/records"
+)
+
+// ISimpleStrategy interface
+type ISimpleStrategy interface {
+	GetTitleAndDescription() (string, string)
+	GetCandidates() []string
+	AddHistoryRecord(record *records.EnrichedRecord) error
+	ResetHistory() error
+}
+
+// IStrategy interface
+type IStrategy interface {
+	GetTitleAndDescription() (string, string)
+	GetCandidates(r records.EnrichedRecord) []string
+	AddHistoryRecord(record *records.EnrichedRecord) error
+	ResetHistory() error
+}
+
+type simpleStrategyWrapper struct {
+	strategy ISimpleStrategy
+}
+
+// NewSimpleStrategyWrapper returns IStrategy created by wrapping given ISimpleStrategy
+func NewSimpleStrategyWrapper(strategy ISimpleStrategy) *simpleStrategyWrapper {
+	return &simpleStrategyWrapper{strategy: strategy}
+}
+
+func (s *simpleStrategyWrapper) GetTitleAndDescription() (string, string) {
+	return s.strategy.GetTitleAndDescription()
+}
+
+func (s *simpleStrategyWrapper) GetCandidates(r records.EnrichedRecord) []string {
+	return s.strategy.GetCandidates()
+}
+
+func (s *simpleStrategyWrapper) AddHistoryRecord(r *records.EnrichedRecord) error {
+	return s.strategy.AddHistoryRecord(r)
+}
+
+func (s *simpleStrategyWrapper) ResetHistory() error {
+	return s.strategy.ResetHistory()
+}
--- a/scripts/install_helper.sh
+++ b/scripts/install_helper.sh
--- a/scripts/rawinstall.sh
+++ b/scripts/rawinstall.sh
--- a/evaluate/resh-evaluate-plot.py
+++ b/evaluate/resh-evaluate-plot.py
@ -9,6 +9,7 @@ import matplotlib.pyplot as plt
 import matplotlib.path as mpath
 import numpy as np
 from graphviz import Digraph
+from datetime import datetime

 PLOT_WIDTH = 10 # inches
 PLOT_HEIGHT = 7 # inches
@ -22,11 +23,11 @@ DATA_records_by_session = defaultdict(list)
 for user in data["UsersRecords"]:
    for device in user["Devices"]:
        for record in device["Records"]:
-            if record["invalid"]:
+            if "invalid" in record and record["invalid"]:
                continue
            
            DATA_records.append(record)
-            DATA_records_by_session[record["sessionId"]].append(record)
+            DATA_records_by_session[record["seqSessionId"]].append(record)

 DATA_records = list(sorted(DATA_records, key=lambda x: x["realtimeAfterLocal"]))

@ -39,7 +40,6 @@ async_draw = True
 # for strategy in data["Strategies"]:
 #     print(json.dumps(strategy))

-
 def zipf(length):
    return list(map(lambda x: 1/2**x, range(0, length)))

@ -81,7 +81,7 @@ def plot_cmdLineFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False):
 def plot_cmdFrq_rank(plotSize=PLOT_SIZE_zipf, show_labels=False):
    cmd_count = defaultdict(int)
    for record in DATA_records:
-        cmd = record["firstWord"]
+        cmd = record["command"]
        if cmd == "":
            continue
        cmd_count[cmd] += 1
@ -111,7 +111,7 @@ def plot_cmdVocabularySize_cmdLinesEntered():
    cmd_vocabulary = set()
    y_cmd_count = [0]
    for record in DATA_records:
-        cmd = record["firstWord"]
+        cmd = record["command"]
        if cmd in cmd_vocabulary:
            # repeat last value
            y_cmd_count.append(y_cmd_count[-1])
@ -163,7 +163,7 @@ def plot_cmdLineVocabularySize_cmdLinesEntered():
 # Figure 3.3. Sequential structure of UNIX command usage, from Figure 4 in Hanson et al. (1984).
 #       Ball diameters are proportional to stationary probability. Lines indicate significant dependencies,
 #       solid ones being more probable (p < .0001) and dashed ones less probable (.005 < p < .0001).
-def graph_cmdSequences(node_count=33, edge_minValue=0.05):
+def graph_cmdSequences(node_count=33, edge_minValue=0.05, view_graph=True):
    START_CMD = "_start_"
    cmd_count = defaultdict(int)
    cmdSeq_count = defaultdict(lambda: defaultdict(int))
@ -174,7 +174,7 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05):
        cmd_count[START_CMD] += 1
        prev_cmd = START_CMD
        for record in session:
-            cmd = record["firstWord"]
+            cmd = record["command"]
            cmdSeq_count[prev_cmd][cmd] += 1
            cmd_count[cmd] += 1
            if cmd not in cmd_id:
@ -265,8 +265,8 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05):

        # graphviz sometimes fails - see above
        try:
-            graph.view()
-            # graph.render('/tmp/resh-graphviz-cmdSeq.gv', view=True)
+            # graph.view()
+            graph.render('/tmp/resh-graph-command_sequence-nodeCount_{}-edgeMinVal_{}.gv'.format(node_count, edge_minValue), view=view_graph)
            break
        except Exception as e:
            trace = traceback.format_exc()
@ -275,7 +275,7 @@ def graph_cmdSequences(node_count=33, edge_minValue=0.05):

 def plot_strategies_matches(plot_size=50, selected_strategies=[]):
    plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT))
-    plt.title("Matches at distance")
+    plt.title("Matches at distance <{}>".format(datetime.now().strftime('%H:%M:%S')))
    plt.ylabel('%' + " of matches")
    plt.xlabel("Distance")
    legend = []
@ -348,10 +348,9 @@ def plot_strategies_matches(plot_size=50, selected_strategies=[]):
        plt.show()


-
 def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]):
    plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT))
-    plt.title("Average characters recalled at distance")
+    plt.title("Average characters recalled at distance <{}>".format(datetime.now().strftime('%H:%M:%S')))
    plt.ylabel("Average characters recalled")
    plt.xlabel("Distance")
    x_values = range(1, plot_size+1)
@ -420,19 +419,101 @@ def plot_strategies_charsRecalled(plot_size=50, selected_strategies=[]):
        plt.show()


+def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]):
+    plt.figure(figsize=(PLOT_WIDTH, PLOT_HEIGHT))
+    plt.title("Average characters recalled at distance (including prefix matches) <{}>".format(datetime.now().strftime('%H:%M:%S'))) 
+    plt.ylabel("Average characters recalled (including prefix matches)")
+    plt.xlabel("Distance")
+    x_values = range(1, plot_size+1)
+    legend = []
+    saved_charsRecalled_total = None
+    saved_dataPoint_count = None
+    for strategy in data["Strategies"]:
+        strategy_title = strategy["Title"]
+        # strategy_description = strategy["Description"]
+
+        dataPoint_count = 0
+        matches_total = 0
+        charsRecalled = [0] * plot_size
+        charsRecalled_total = 0
        
-# graph_cmdSequences(node_count=33, edge_minValue=0.05)
-graph_cmdSequences(node_count=28, edge_minValue=0.06)
+        for multiMatch in strategy["PrefixMatches"]:
+            dataPoint_count += 1
+
+            if not multiMatch["Match"]:
+                continue
+            matches_total += 1
+
+            last_charsRecalled = 0
+            for match in multiMatch["Entries"]:
+
+                chars = match["CharsRecalled"]
+                charsIncrease = chars - last_charsRecalled
+                assert(charsIncrease > 0)
+                charsRecalled_total += charsIncrease 
+
+                dist = match["Distance"]  
+                if dist > plot_size:
+                    continue
+
+                charsRecalled[dist-1] += charsIncrease
+                last_charsRecalled = chars
+            
+        # recent is very simple strategy so we will believe 
+        #       that there is no bug in it and we can use it to determine total
+        if strategy_title == "recent":
+            saved_charsRecalled_total = charsRecalled_total
+            saved_dataPoint_count = dataPoint_count
+
+        if len(selected_strategies) and strategy_title not in selected_strategies:
+            continue
+
+        acc = 0
+        charsRecalled_cumulative = []
+        for x in charsRecalled:
+            acc += x
+            charsRecalled_cumulative.append(acc)
+        charsRecalled_average = list(map(lambda x: x / dataPoint_count, charsRecalled_cumulative))
+
+        plt.plot(x_values, charsRecalled_average, 'o-')
+        legend.append(strategy_title)
+
+    assert(saved_charsRecalled_total is not None)
+    assert(saved_dataPoint_count is not None)
+    max_values = [saved_charsRecalled_total / saved_dataPoint_count] * len(x_values)
+    plt.plot(x_values, max_values, 'r-')
+    legend.append("maximum possible")
+
+    x_ticks = list(range(1, plot_size+1, 2))
+    x_labels = x_ticks[:]
+    plt.xticks(x_ticks, x_labels)
+    plt.legend(legend, loc="best")
+    if async_draw:
+        plt.draw()
+    else:
+        plt.show()
+

-plot_cmdLineFrq_rank()
-plot_cmdFrq_rank()
+# plot_cmdLineFrq_rank()
+# plot_cmdFrq_rank()
        
-plot_cmdLineVocabularySize_cmdLinesEntered()
-plot_cmdVocabularySize_cmdLinesEntered()
+# plot_cmdLineVocabularySize_cmdLinesEntered()
+# plot_cmdVocabularySize_cmdLinesEntered()

 plot_strategies_matches(20)
 plot_strategies_charsRecalled(20)
+# plot_strategies_charsRecalled_prefix(20)
+
+# graph_cmdSequences(node_count=33, edge_minValue=0.048)
+
+# graph_cmdSequences(node_count=28, edge_minValue=0.06)
+
+# for n in range(29, 35):
+#     for e in range(44, 56, 2):
+#         e *= 0.001
+#         graph_cmdSequences(node_count=n, edge_minValue=e, view_graph=False)
+
+# be careful and check if labels fit the display

 if async_draw:
    plt.show()
-# be careful and check if labels fit the display
--- a/scripts/shellrc.sh
+++ b/scripts/shellrc.sh
--- a/scripts/test.sh
+++ b/scripts/test.sh
@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+[ "${BASH_SOURCE[0]}" != "scripts/test.sh" ] && echo 'Run this script using `make test`' && exit 1 
+
+for f in scripts/*.sh; do
+    echo "Running shellcheck on $f ..."
+    shellcheck $f --shell=bash --severity=error || exit 1
+done
+
+echo "Checking Zsh syntax of scripts/shellrc.sh ..."
+! zsh -n scripts/shellrc.sh && echo "Zsh syntax check failed!" && exit 1
+
+for sh in bash zsh; do
+    echo "Running functions in scripts/shellrc.sh using $sh ..."
+    ! $sh -c ". scripts/shellrc.sh; __resh_preexec; __resh_precmd" && echo "Error while running functions!" && exit 1
+done
+
+# TODO: test installation
+
+exit 0
--- a/scripts/uuid.sh
+++ b/scripts/uuid.sh