evaluate: add dynamic/tf-idf record distance strategy

evaluate/resh-evaluate-plot.py

@@ -502,7 +502,7 @@ def plot_strategies_charsRecalled_prefix(plot_size=50, selected_strategies=[]):
 
 plot_strategies_matches(20)
 plot_strategies_charsRecalled(20)
-plot_strategies_charsRecalled_prefix(20)
+# plot_strategies_charsRecalled_prefix(20)
 
 # graph_cmdSequences(node_count=33, edge_minValue=0.048)
 

evaluate/resh-evaluate.go

@@ -114,13 +114,23 @@ func main() {
 	directory.init()
 	strategies = append(strategies, &directory)
 
-	if *slow {
+	dynamicDist := strategyDynamicRecordDistance{
+		maxDepth:   3000,
+		distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1},
+		label:      "10*pwd,10*realpwd,session,time",
+	}
+	dynamicDist.init()
+	strategies = append(strategies, &dynamicDist)
+
 	distanceStaticBest := strategyRecordDistance{
-			distParams: common.DistParams{SessionID: 1, Pwd: 10, RealPwd: 10, Time: 1},
-			label:      "10*pwd,10*realpwd,1*session,time",
+		maxDepth:   3000,
+		distParams: common.DistParams{Pwd: 10, RealPwd: 10, SessionID: 1, Time: 1},
+		label:      "10*pwd,10*realpwd,session,time",
 	}
 	strategies = append(strategies, &distanceStaticBest)
 
+	if *slow {
+
 		markovCmd := strategyMarkovChainCmd{order: 1}
 		markovCmd.init()
 

evaluate/strategy-dynamic-record-distance.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"math"
+	"sort"
+	"strconv"
+
+	"github.com/curusarn/resh/common"
+)
+
+type strategyDynamicRecordDistance struct {
+	history          []common.EnrichedRecord
+	distParams       common.DistParams
+	pwdHistogram     map[string]int
+	realPwdHistogram map[string]int
+	maxDepth         int
+	label            string
+}
+
+type strDynDistEntry struct {
+	cmdLine  string
+	distance float64
+}
+
+func (s *strategyDynamicRecordDistance) init() {
+	s.history = nil
+	s.pwdHistogram = map[string]int{}
+	s.realPwdHistogram = map[string]int{}
+}
+
+func (s *strategyDynamicRecordDistance) GetTitleAndDescription() (string, string) {
+	return "dynamic record distance (depth:" + strconv.Itoa(s.maxDepth) + ";" + s.label + ")", "Use TF-IDF record distance to recommend commands"
+}
+
+func (s *strategyDynamicRecordDistance) idf(count int) float64 {
+	return math.Log(float64(len(s.history)) / float64(count))
+}
+
+func (s *strategyDynamicRecordDistance) GetCandidates() []string {
+	if len(s.history) == 0 {
+		return nil
+	}
+	var prevRecord common.EnrichedRecord
+	prevRecord = s.history[0]
+	prevRecord.SetCmdLine("")
+	prevRecord.SetBeforeToAfter()
+	var mapItems []strDynDistEntry
+	for i, record := range s.history {
+		if s.maxDepth != 0 && i > s.maxDepth {
+			break
+		}
+		distParams := common.DistParams{
+			Pwd:       s.distParams.Pwd * s.idf(s.pwdHistogram[prevRecord.PwdAfter]),
+			RealPwd:   s.distParams.RealPwd * s.idf(s.realPwdHistogram[prevRecord.RealPwdAfter]),
+			Time:      s.distParams.Time,
+			SessionID: s.distParams.SessionID,
+		}
+		distance := record.DistanceTo(prevRecord, distParams)
+		mapItems = append(mapItems, strDynDistEntry{record.CmdLine, distance})
+	}
+	sort.SliceStable(mapItems, func(i int, j int) bool { return mapItems[i].distance < mapItems[j].distance })
+	var hist []string
+	histSet := map[string]bool{}
+	for _, item := range mapItems {
+		if histSet[item.cmdLine] {
+			continue
+		}
+		histSet[item.cmdLine] = true
+		hist = append(hist, item.cmdLine)
+	}
+	return hist
+}
+
+func (s *strategyDynamicRecordDistance) AddHistoryRecord(record *common.EnrichedRecord) error {
+	// append record to front
+	s.history = append([]common.EnrichedRecord{*record}, s.history...)
+	s.pwdHistogram[record.Pwd]++
+	s.realPwdHistogram[record.RealPwd]++
+	return nil
+}
+
+func (s *strategyDynamicRecordDistance) ResetHistory() error {
+	s.init()
+	return nil
+}