Restructure project to match guidelines

.gitignore

@@ -1,4 +1 @@
-resh-collect
-resh-daemon
-resh-sanitize-history
-resh-evaluate
+bin/*

Makefile

@@ -1,10 +1,10 @@
 SHELL=/bin/bash
-VERSION=$(shell cat version)
+VERSION=$(shell cat VERSION)
 REVISION=$(shell [ -z "$(git status --untracked-files=no --porcelain)" ] && git rev-parse --short=12 HEAD || echo "no_revision")
 GOFLAGS=-ldflags "-X main.Version=${VERSION} -X main.Revision=${REVISION}"
 
 autoinstall: 
-	./install_helper.sh
+	scripts/install_helper.sh
 
 sanitize:
 	#
@@ -23,8 +23,8 @@ sanitize:
 	#
 	#
 	# Running history sanitization ...
-	resh-sanitize-history -trim-hashes 0 --output ~/resh_history_sanitized.json
-	resh-sanitize-history -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json
+	resh-sanitize -trim-hashes 0 --output ~/resh_history_sanitized.json
+	resh-sanitize -trim-hashes 12 --output ~/resh_history_sanitized_trim12.json
 	# 
 	# 
 	# SUCCESS - ALL DONE!
@@ -41,8 +41,7 @@ sanitize:
 	#
 	#
 
-
-build: submodules resh-collect resh-daemon resh-sanitize-history resh-evaluate
+build: submodules bin/resh-collect bin/resh-daemon bin/resh-evaluate bin/resh-sanitize
 
 rebuild:
 	make clean
@@ -51,15 +50,15 @@ rebuild:
 clean:
 	rm resh-*
 
-install: build submodules/bash-preexec/bash-preexec.sh shellrc.sh config.toml uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config
+install: build submodules/bash-preexec/bash-preexec.sh scripts/shellrc.sh conf/config.toml scripts/uuid.sh | $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config
 	# Copying files to resh directory ...
 	cp -f submodules/bash-preexec/bash-preexec.sh ~/.bash-preexec.sh
-	cp -f config.toml ~/.config/resh.toml
-	cp -f shellrc.sh ~/.resh/shellrc
-	cp -f uuid.sh ~/.resh/bin/resh-uuid
-	cp -f resh-* ~/.resh/bin/
-	cp -f evaluate/resh-evaluate-plot.py ~/.resh/bin/
-	cp -fr sanitizer_data ~/.resh/
+	cp -f conf/config.toml ~/.config/resh.toml
+	cp -f scripts/shellrc.sh ~/.resh/shellrc
+	cp -f scripts/uuid.sh ~/.resh/bin/resh-uuid
+	cp -f bin/* ~/.resh/bin/
+	cp -f scripts/resh-evaluate-plot.py ~/.resh/bin/
+	cp -fr data/sanitizer ~/.resh/
 	# backward compatibility: We have a new location for resh history file 
 	[ ! -f ~/.resh/history.json ] || mv ~/.resh/history.json ~/.resh_history.json 
 	# Adding resh shellrc to .bashrc ...
@@ -107,17 +106,8 @@ uninstall:
 	# Uninstalling ...
 	-rm -rf ~/.resh/
 
-resh-daemon: daemon/resh-daemon.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $<
-
-resh-collect: collect/resh-collect.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $<
-
-resh-sanitize-history: sanitize-history/resh-sanitize-history.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $<
-
-resh-evaluate: evaluate/resh-evaluate.go evaluate/strategy-*.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $< evaluate/strategy-*.go 
+bin/resh-%: cmd/%/main.go pkg/*/*.go VERSION
+	go build ${GOFLAGS} -o $@ cmd/$*/*.go
 
 $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config:
 	# Creating dirs ...
@@ -129,7 +119,6 @@ $(HOME)/.resh/resh-uuid:
 
 .PHONY: submodules build install rebuild uninstall clean autoinstall
 
-
 submodules: | submodules/bash-preexec/bash-preexec.sh
 	@# sets submodule.recurse to true if unset
 	@# sets status.submoduleSummary to true if unset

README.md

@@ -17,6 +17,7 @@ If you are not happy with it you can uninstall it with a single command (`rm -rf
 The ultimate point of my thesis is to provide a context-based replacement/enhancement for bash and zsh shell history.
 
 The idea is to:
+
 - Save each command with metadata (device, directory, git, time, terminal session pid, ... see example below)
 - Recommend history based on saved metadata
   - e.g. it will be easier to get to commands specific to the project you are currently working on (based on directory, git repository url, ...)
@@ -45,9 +46,11 @@ If you install RESH, please give me some contact info using this form: https://f
 ## Installation
 
 ### Simplest
-Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/rawinstall.sh)"` from anywhere.
+
+Just run `bash -c "$(wget -O - https://raw.githubusercontent.com/curusarn/resh/master/scripts/rawinstall.sh)"` from anywhere.
 
 ### Simple
+
 1. Run `git clone https://github.com/curusarn/resh.git && cd resh`
 2. Run `make autoinstall` for assisted build & instalation.
     - OR Run `make install` if you know how to build Golang projects.
@@ -59,6 +62,7 @@ If you install RESH, please give me some contact info using this form: https://f
 Works in `bash` and `zsh`.
 
 Tested on:
+
 - Arch
 - MacOS
 - Ubuntu (18.04)

cmd/collect/main.go

@@ -11,7 +11,7 @@ import (
 	"os"
 
 	"github.com/BurntSushi/toml"
-	common "github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 
 	//  "os/exec"
 	"os/user"
@@ -34,7 +34,7 @@ func main() {
 
 	machineIDPath := "/etc/machine-id"
 
-	var config common.Config
+	var config records.Config
 	if _, err := toml.DecodeFile(configPath, &config); err != nil {
 		log.Fatal("Error reading config:", err)
 	}
@@ -171,12 +171,12 @@ func main() {
 		*osReleasePrettyName = "Linux"
 	}
 
-	rec := common.Record{
+	rec := records.Record{
 		// posix
 		Cols:  *cols,
 		Lines: *lines,
 		// core
-		BaseRecord: common.BaseRecord{
+		BaseRecord: records.BaseRecord{
 			CmdLine:   *cmdLine,
 			ExitCode:  *exitCode,
 			Shell:     *shell,
@@ -237,7 +237,7 @@ func main() {
 	sendRecord(rec, strconv.Itoa(config.Port))
 }
 
-func sendRecord(r common.Record, port string) {
+func sendRecord(r records.Record, port string) {
 	recJSON, err := json.Marshal(r)
 	if err != nil {
 		log.Fatal("send err 1", err)

cmd/daemon/main.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
-	common "github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 )
 
 // Version from git set during build
@@ -43,7 +43,7 @@ func main() {
 	log.SetOutput(f)
 	log.SetPrefix(strconv.Itoa(os.Getpid()) + " | ")
 
-	var config common.Config
+	var config records.Config
 	if _, err := toml.DecodeFile(configPath, &config); err != nil {
 		log.Println("Error reading config", err)
 		return
@@ -88,7 +88,7 @@ type recordHandler struct {
 
 func (h *recordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("OK\n"))
-	record := common.Record{}
+	record := records.Record{}
 
 	jsn, err := ioutil.ReadAll(r.Body)
 	if err != nil {

cmd/evaluate/main.go

@@ -15,8 +15,9 @@ import (
 	"path/filepath"
 	"sort"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 	"github.com/jpillora/longestcommon"
+
 	"github.com/schollz/progressbar"
 )
 
@@ -116,7 +117,7 @@ func main() {
 
 	dynamicDist := strategyDynamicRecordDistance{
 		maxDepth:   3000,
-		distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1},
+		distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1},
 		label:      "10*pwd,10*realpwd,session,time",
 	}
 	dynamicDist.init()
@@ -124,7 +125,7 @@ func main() {
 
 	distanceStaticBest := strategyRecordDistance{
 		maxDepth:   3000,
-		distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1},
+		distParams: records.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1},
 		label:      "10*pwd,10*realpwd,session,time",
 	}
 	strategies = append(strategies, &distanceStaticBest)
@@ -159,7 +160,7 @@ func main() {
 type strategy interface {
 	GetTitleAndDescription() (string, string)
 	GetCandidates() []string
-	AddHistoryRecord(record *common.EnrichedRecord) error
+	AddHistoryRecord(record *records.EnrichedRecord) error
 	ResetHistory() error
 }
 
@@ -188,7 +189,7 @@ type strategyJSON struct {
 
 type deviceRecords struct {
 	Name    string
-	Records []common.EnrichedRecord
+	Records []records.EnrichedRecord
 }
 
 type userRecords struct {
@@ -284,7 +285,7 @@ func (e *evaluator) evaluate(strategy strategy) error {
 	for i := range e.UsersRecords {
 		for j := range e.UsersRecords[i].Devices {
 			bar := progressbar.New(len(e.UsersRecords[i].Devices[j].Records))
-			var prevRecord common.EnrichedRecord
+			var prevRecord records.EnrichedRecord
 			for _, record := range e.UsersRecords[i].Devices[j].Records {
 				if e.skipFailedCmds && record.ExitCode != 0 {
 					continue
@@ -416,18 +417,18 @@ func (e *evaluator) loadHistoryRecordsBatchMode(fname string, dataRootPath strin
 	return records
 }
 
-func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord {
+func (e *evaluator) loadHistoryRecords(fname string) []records.EnrichedRecord {
 	file, err := os.Open(fname)
 	if err != nil {
 		log.Fatal("Open() resh history file error:", err)
 	}
 	defer file.Close()
 
-	var records []common.EnrichedRecord
+	var recs []records.EnrichedRecord
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		record := common.Record{}
-		fallbackRecord := common.FallbackRecord{}
+		record := records.Record{}
+		fallbackRecord := records.FallbackRecord{}
 		line := scanner.Text()
 		err = json.Unmarshal([]byte(line), &record)
 		if err != nil {
@@ -436,7 +437,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord {
 				log.Println("Line:", line)
 				log.Fatal("Decoding error:", err)
 			}
-			record = common.ConvertRecord(&fallbackRecord)
+			record = records.ConvertRecord(&fallbackRecord)
 		}
 		if e.sanitizedInput == false {
 			if record.CmdLength != 0 {
@@ -447,7 +448,7 @@ func (e *evaluator) loadHistoryRecords(fname string) []common.EnrichedRecord {
 		if record.CmdLength == 0 {
 			log.Fatal("Assert failed - 'cmdLength' is unset in the data. This should not happen.")
 		}
-		records = append(records, record.Enrich())
+		recs = append(recs, record.Enrich())
 	}
-	return records
+	return recs
 }

cmd/evaluate/strategy-directory-sensitive.go

@@ -1,6 +1,6 @@
 package main
 
-import "github.com/curusarn/resh/common"
+import "github.com/curusarn/resh/pkg/records"
 
 type strategyDirectorySensitive struct {
 	history map[string][]string
@@ -19,7 +19,7 @@ func (s *strategyDirectorySensitive) GetCandidates() []string {
 	return s.history[s.lastPwd]
 }
 
-func (s *strategyDirectorySensitive) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyDirectorySensitive) AddHistoryRecord(record *records.EnrichedRecord) error {
 	// work on history for PWD
 	pwd := record.Pwd
 	// remove previous occurance of record

cmd/evaluate/strategy-dummy.go

@@ -1,6 +1,6 @@
 package main
 
-import "github.com/curusarn/resh/common"
+import "github.com/curusarn/resh/pkg/records"
 
 type strategyDummy struct {
 	history []string
@@ -14,7 +14,7 @@ func (s *strategyDummy) GetCandidates() []string {
 	return nil
 }
 
-func (s *strategyDummy) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyDummy) AddHistoryRecord(record *records.EnrichedRecord) error {
 	s.history = append(s.history, record.CmdLine)
 	return nil
 }

cmd/evaluate/strategy-dynamic-record-distance.go

@@ -5,12 +5,12 @@ import (
 	"sort"
 	"strconv"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 )
 
 type strategyDynamicRecordDistance struct {
-	history          []common.EnrichedRecord
-	distParams       common.DistParams
+	history          []records.EnrichedRecord
+	distParams       records.DistParams
 	pwdHistogram     map[string]int
 	realPwdHistogram map[string]int
 	maxDepth         int
@@ -40,7 +40,7 @@ func (s *strategyDynamicRecordDistance) GetCandidates() []string {
 	if len(s.history) == 0 {
 		return nil
 	}
-	var prevRecord common.EnrichedRecord
+	var prevRecord records.EnrichedRecord
 	prevRecord = s.history[0]
 	prevRecord.SetCmdLine("")
 	prevRecord.SetBeforeToAfter()
@@ -49,7 +49,7 @@ func (s *strategyDynamicRecordDistance) GetCandidates() []string {
 		if s.maxDepth != 0 && i > s.maxDepth {
 			break
 		}
-		distParams := common.DistParams{
+		distParams := records.DistParams{
 			Pwd:       s.distParams.Pwd * s.idf(s.pwdHistogram[prevRecord.PwdAfter]),
 			RealPwd:   s.distParams.RealPwd * s.idf(s.realPwdHistogram[prevRecord.RealPwdAfter]),
 			Time:      s.distParams.Time,
@@ -71,9 +71,9 @@ func (s *strategyDynamicRecordDistance) GetCandidates() []string {
 	return hist
 }
 
-func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error {
 	// append record to front
-	s.history = append([]common.EnrichedRecord{*record}, s.history...)
+	s.history = append([]records.EnrichedRecord{*record}, s.history...)
 	s.pwdHistogram[record.Pwd]++
 	s.realPwdHistogram[record.RealPwd]++
 	return nil

cmd/evaluate/strategy-frequent.go

@@ -3,7 +3,7 @@ package main
 import (
 	"sort"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 )
 
 type strategyFrequent struct {
@@ -36,7 +36,7 @@ func (s *strategyFrequent) GetCandidates() []string {
 	return hist
 }
 
-func (s *strategyFrequent) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyFrequent) AddHistoryRecord(record *records.EnrichedRecord) error {
 	s.history[record.CmdLine]++
 	return nil
 }

cmd/evaluate/strategy-markov-chain-cmd.go

@@ -4,7 +4,7 @@ import (
 	"sort"
 	"strconv"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 	"github.com/mb-14/gomarkov"
 )
 
@@ -78,7 +78,7 @@ func (s *strategyMarkovChainCmd) GetCandidates() []string {
 	return hist
 }
 
-func (s *strategyMarkovChainCmd) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyMarkovChainCmd) AddHistoryRecord(record *records.EnrichedRecord) error {
 	s.history = append(s.history, strMarkCmdHistoryEntry{cmdLine: record.CmdLine, cmd: record.Command})
 	s.historyCmds = append(s.historyCmds, record.Command)
 	// s.historySet[record.CmdLine] = true

cmd/evaluate/strategy-markov-chain.go

@@ -4,7 +4,7 @@ import (
 	"sort"
 	"strconv"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 	"github.com/mb-14/gomarkov"
 )
 
@@ -58,7 +58,7 @@ func (s *strategyMarkovChain) GetCandidates() []string {
 	return hist
 }
 
-func (s *strategyMarkovChain) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyMarkovChain) AddHistoryRecord(record *records.EnrichedRecord) error {
 	s.history = append(s.history, record.CmdLine)
 	// s.historySet[record.CmdLine] = true
 	return nil

cmd/evaluate/strategy-random.go

@@ -4,7 +4,7 @@ import (
 	"math/rand"
 	"time"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 )
 
 type strategyRandom struct {
@@ -39,7 +39,7 @@ func (s *strategyRandom) GetCandidates() []string {
 	return candidates
 }
 
-func (s *strategyRandom) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyRandom) AddHistoryRecord(record *records.EnrichedRecord) error {
 	s.history = append([]string{record.CmdLine}, s.history...)
 	s.historySet[record.CmdLine] = true
 	return nil

cmd/evaluate/strategy-recent.go

@@ -1,6 +1,6 @@
 package main
 
-import "github.com/curusarn/resh/common"
+import "github.com/curusarn/resh/pkg/records"
 
 type strategyRecent struct {
 	history []string
@@ -14,7 +14,7 @@ func (s *strategyRecent) GetCandidates() []string {
 	return s.history
 }
 
-func (s *strategyRecent) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyRecent) AddHistoryRecord(record *records.EnrichedRecord) error {
 	// remove previous occurance of record
 	for i, cmd := range s.history {
 		if cmd == record.CmdLine {

cmd/evaluate/strategy-record-distance.go

@@ -4,12 +4,12 @@ import (
 	"sort"
 	"strconv"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 )
 
 type strategyRecordDistance struct {
-	history    []common.EnrichedRecord
-	distParams common.DistParams
+	history    []records.EnrichedRecord
+	distParams records.DistParams
 	maxDepth   int
 	label      string
 }
@@ -31,7 +31,7 @@ func (s *strategyRecordDistance) GetCandidates() []string {
 	if len(s.history) == 0 {
 		return nil
 	}
-	var prevRecord common.EnrichedRecord
+	var prevRecord records.EnrichedRecord
 	prevRecord = s.history[0]
 	prevRecord.SetCmdLine("")
 	prevRecord.SetBeforeToAfter()
@@ -56,9 +56,9 @@ func (s *strategyRecordDistance) GetCandidates() []string {
 	return hist
 }
 
-func (s *strategyRecordDistance) AddHistoryRecord(record *common.EnrichedRecord) error {
+func (s *strategyRecordDistance) AddHistoryRecord(record *records.EnrichedRecord) error {
 	// append record to front
-	s.history = append([]common.EnrichedRecord{*record}, s.history...)
+	s.history = append([]records.EnrichedRecord{*record}, s.history...)
 	return nil
 }
 

cmd/sanitize/main.go

@@ -19,7 +19,7 @@ import (
 	"strings"
 	"unicode"
 
-	"github.com/curusarn/resh/common"
+	"github.com/curusarn/resh/pkg/records"
 	giturls "github.com/whilp/git-urls"
 )
 
@@ -79,8 +79,8 @@ func main() {
 
 	scanner := bufio.NewScanner(inputFile)
 	for scanner.Scan() {
-		record := common.Record{}
-		fallbackRecord := common.FallbackRecord{}
+		record := records.Record{}
+		fallbackRecord := records.FallbackRecord{}
 		line := scanner.Text()
 		err = json.Unmarshal([]byte(line), &record)
 		if err != nil {
@@ -89,7 +89,7 @@ func main() {
 				log.Println("Line:", line)
 				log.Fatal("Decoding error:", err)
 			}
-			record = common.ConvertRecord(&fallbackRecord)
+			record = records.ConvertRecord(&fallbackRecord)
 		}
 		err = sanitizer.sanitizeRecord(&record)
 		if err != nil {
@@ -139,7 +139,7 @@ func loadData(fname string) map[string]bool {
 	return data
 }
 
-func (s *sanitizer) sanitizeRecord(record *common.Record) error {
+func (s *sanitizer) sanitizeRecord(record *records.Record) error {
 	// hash directories of the paths
 	record.Pwd = s.sanitizePath(record.Pwd)
 	record.RealPwd = s.sanitizePath(record.RealPwd)

pkg/records/records.go

@@ -1,4 +1,4 @@
-package common
+package records
 
 import (
 	"encoding/json"
@@ -246,7 +246,7 @@ func GetCommandAndFirstWord(cmdLine string) (string, string, error) {
 	return "ERROR", "ERROR", errors.New("this should not happen - contact developer ;)")
 }
 
-// DistParams is used to supply params to EnrichedRecord.DistanceTo()
+// DistParams is used to supply params to Enrichedrecords.DistanceTo()
 type DistParams struct {
 	ExitCode  float64
 	MachineID float64