checkpoint before separating plotting into python

because matplotlib is just too good and trying to plot in golang is not worth it

Makefile

@@ -115,8 +115,8 @@ resh-collect: collect/resh-collect.go common/resh-common.go version
 resh-sanitize-history: sanitize-history/resh-sanitize-history.go common/resh-common.go version
 	go build ${GOFLAGS} -o $@ $<
 
-resh-evaluate: evaluate/resh-evaluate.go evaluate/statistics.go evaluate/strategy-*.go common/resh-common.go version
+resh-evaluate: evaluate/resh-evaluate.go evaluate/results.go evaluate/statistics.go evaluate/strategy-*.go common/resh-common.go version
-	go build ${GOFLAGS} -o $@ $< evaluate/statistics.go evaluate/strategy-*.go 
+	go build ${GOFLAGS} -o $@ $< evaluate/results.go evaluate/statistics.go evaluate/strategy-*.go 
 
 $(HOME)/.resh $(HOME)/.resh/bin $(HOME)/.config:
 	# Creating dirs ...

evaluate/resh-evaluate.go

@@ -108,12 +108,15 @@ func (e *evaluator) init(inputPath string) error {
 	return nil
 }
 
-func (e *evaluator) evaluate(strat strategy) error {
+func (e *evaluator) evaluate(strategy strategy) error {
-	stats := statistics{writer: e.writer, size: e.maxCandidates + 1}
+	res := results{writer: e.writer, size: e.maxCandidates + 1}
+	stats := statistics{}
+	res.init()
 	stats.init()
 
 	for _, record := range e.historyRecords {
-		candidates := strat.GetCandidates()
+		stats.addCmdLine(record.CmdLine, record.CmdLength)
+		candidates := strategy.GetCandidates()
 
 		match := false
 		for i, candidate := range candidates {
@@ -122,21 +125,21 @@ func (e *evaluator) evaluate(strat strategy) error {
 			// 	break
 			// }
 			if candidate == record.CmdLine {
-				stats.addMatch(i+1, record.CmdLength)
+				res.addMatch(i+1, record.CmdLength)
 				match = true
 				break
 			}
 		}
 		if match == false {
-			stats.addMiss()
+			res.addMiss()
 		}
-		err := strat.AddHistoryRecord(&record)
+		err := strategy.AddHistoryRecord(&record)
 		if err != nil {
 			log.Println("Error while evauating", err)
 			return err
 		}
 	}
-	title, description := strat.GetTitleAndDescription()
+	title, description := strategy.GetTitleAndDescription()
 	n, err := e.writer.WriteString(title + " - " + description + "\n")
 	if err != nil {
 		log.Fatal(err)
@@ -145,7 +148,8 @@ func (e *evaluator) evaluate(strat strategy) error {
 		log.Fatal("Nothing was written", n)
 	}
 	// print results
-	stats.printCumulative()
+	res.printCumulative()
+	stats.graphCmdFrequencyAsFuncOfRank()
 	return nil
 }
 

evaluate/results.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"math"
+	"strconv"
+)
+
+type results struct {
+	writer                  *bufio.Writer
+	size                    int
+	matches                 []int // matches[N] -> # of matches at distance N
+	matchesTotal            int
+	charactersRecalled      []int
+	charactersRecalledTotal int
+	dataPointCount          int
+}
+
+func (r *results) init() {
+	r.matches = make([]int, r.size)
+	r.charactersRecalled = make([]int, r.size)
+}
+
+func (r *results) addMatch(distance int, cmdLength int) {
+	if distance >= r.size {
+		// --calculate-total
+		// log.Fatal("Match distance is greater than size of statistics")
+		r.matchesTotal++
+		r.charactersRecalledTotal += cmdLength
+		return
+	}
+	r.matches[distance]++
+	r.matchesTotal++
+	r.charactersRecalled[distance] += cmdLength
+	r.charactersRecalledTotal += cmdLength
+	r.dataPointCount++
+}
+
+func (r *results) addMiss() {
+	r.dataPointCount++
+}
+
+func (r *results) printCumulative() {
+	matchesPercent := 0.0
+	out := "### Matches ###\n"
+	for i := 0; i < r.size; i++ {
+		matchesPercent += 100 * float64(r.matches[i]) / float64(r.dataPointCount)
+		out += strconv.Itoa(i) + " ->"
+		out += fmt.Sprintf(" (%.1f %%)\n", matchesPercent)
+		for j := 0; j < int(math.Round(matchesPercent)); j++ {
+			out += "#"
+		}
+		out += "\n"
+	}
+	matchesPercent = 100 * float64(r.matchesTotal) / float64(r.dataPointCount)
+	out += "TOTAL ->"
+	out += fmt.Sprintf(" (%.1f %%)\n", matchesPercent)
+	for j := 0; j < int(math.Round(matchesPercent)); j++ {
+		out += "#"
+	}
+	out += "\n"
+
+	n, err := r.writer.WriteString(string(out) + "\n\n")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if n == 0 {
+		log.Fatal("Nothing was written", n)
+	}
+
+	charsRecall := 0.0
+	out = "### Characters recalled per submission ###\n"
+	for i := 0; i < r.size; i++ {
+		charsRecall += float64(r.charactersRecalled[i]) / float64(r.dataPointCount)
+		out += strconv.Itoa(i) + " ->"
+		out += fmt.Sprintf(" (%.2f)\n", charsRecall)
+		for j := 0; j < int(math.Round(charsRecall)); j++ {
+			out += "#"
+		}
+		out += "\n"
+	}
+	charsRecall = float64(r.charactersRecalledTotal) / float64(r.dataPointCount)
+	out += "TOTAL ->"
+	out += fmt.Sprintf(" (%.2f)\n", charsRecall)
+	for j := 0; j < int(math.Round(charsRecall)); j++ {
+		out += "#"
+	}
+	out += "\n"
+
+	n, err = r.writer.WriteString(string(out) + "\n\n")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if n == 0 {
+		log.Fatal("Nothing was written", n)
+	}
+}

evaluate/statistics.go

@@ -1,99 +1,117 @@
 package main
 
 import (
-	"bufio"
+	"bytes"
-	"fmt"
+	"io/ioutil"
 	"log"
-	"math"
+	"sort"
-	"strconv"
+
+	"github.com/wcharczuk/go-chart"
 )
 
 type statistics struct {
-	writer                  *bufio.Writer
+	//size                    int
-	size                    int
-	matches                 []int
-	matchesTotal            int
-	charactersRecalled      []int
-	charactersRecalledTotal int
 	dataPointCount int
+	cmdLineCount   map[string]int
 }
 
 func (s *statistics) init() {
-	s.matches = make([]int, s.size)
+	s.cmdLineCount = make(map[string]int)
-	s.charactersRecalled = make([]int, s.size)
 }
 
-func (s *statistics) addMatch(distance int, cmdLength int) {
+func (s *statistics) addCmdLine(cmdLine string, cmdLength int) {
-	if distance >= s.size {
+	s.cmdLineCount[cmdLine]++
-		// --calculate-total
-		// log.Fatal("Match distance is greater than size of statistics")
-		s.matchesTotal++
-		s.charactersRecalledTotal += cmdLength
-		return
-	}
-	s.matches[distance]++
-	s.matchesTotal++
-	s.charactersRecalled[distance] += cmdLength
-	s.charactersRecalledTotal += cmdLength
 	s.dataPointCount++
 }
 
-func (s *statistics) addMiss() {
+func (s *statistics) graphCmdFrequencyAsFuncOfRank() {
-	s.dataPointCount++
-}
 
-func (s *statistics) printCumulative() {
+	var xValues []float64
-	matchesPercent := 0.0
+	var yValues []float64
-	out := "### Matches ###\n"
-	for i := 0; i < s.size; i++ {
-		matchesPercent += 100 * float64(s.matches[i]) / float64(s.dataPointCount)
-		out += strconv.Itoa(i) + " ->"
-		out += fmt.Sprintf(" (%.1f %%)\n", matchesPercent)
-		for j := 0; j < int(math.Round(matchesPercent)); j++ {
-			out += "#"
-		}
-		out += "\n"
-	}
-	matchesPercent = 100 * float64(s.matchesTotal) / float64(s.dataPointCount)
-	out += "TOTAL ->"
-	out += fmt.Sprintf(" (%.1f %%)\n", matchesPercent)
-	for j := 0; j < int(math.Round(matchesPercent)); j++ {
-		out += "#"
-	}
-	out += "\n"
 
-	n, err := s.writer.WriteString(string(out) + "\n\n")
+	sortedValues := sortMapByvalue(s.cmdLineCount)
-	if err != nil {
+	sortedValues = sortedValues[:100] // cut off at rank 100
-		log.Fatal(err)
-	}
-	if n == 0 {
-		log.Fatal("Nothing was written", n)
-	}
 
-	charsRecall := 0.0
+	normalizeCoeficient := float64(s.dataPointCount) / float64(sortedValues[0].Value)
-	out = "### Characters recalled per submission ###\n"
+	for i, pair := range sortedValues {
-	for i := 0; i < s.size; i++ {
+		rank := i + 1
-		charsRecall += float64(s.charactersRecalled[i]) / float64(s.dataPointCount)
+		frequency := float64(pair.Value) / float64(s.dataPointCount)
-		out += strconv.Itoa(i) + " ->"
+		normalizeFrequency := frequency * normalizeCoeficient
-		out += fmt.Sprintf(" (%.2f)\n", charsRecall)
+
-		for j := 0; j < int(math.Round(charsRecall)); j++ {
+		xValues = append(xValues, float64(rank))
-			out += "#"
+		yValues = append(yValues, normalizeFrequency)
-		}
-		out += "\n"
 	}
-	charsRecall = float64(s.charactersRecalledTotal) / float64(s.dataPointCount)
+
-	out += "TOTAL ->"
+	graphName := "cmdFrqAsFuncOfRank"
-	out += fmt.Sprintf(" (%.2f)\n", charsRecall)
+	graph := chart.Chart{
-	for j := 0; j < int(math.Round(charsRecall)); j++ {
+		XAxis: chart.XAxis{
-		out += "#"
+			Style: chart.StyleShow(), //enables / displays the x-axis
+			Ticks: []chart.Tick{
+				{0.0, "0"},
+				{1.0, "1"},
+				{2.0, "2"},
+				{3.0, "3"},
+				{4.0, "4"},
+				{5.0, "5"},
+				{10.0, "10"},
+				{15.0, "15"},
+				{20.0, "20"},
+				{25.0, "25"},
+				{30.0, "30"},
+				{35.0, "35"},
+				{40.0, "40"},
+				{45.0, "45"},
+				{50.0, "50"},
+			},
+		},
+		YAxis: chart.YAxis{
+			AxisType: chart.YAxisSecondary,
+			Style:    chart.StyleShow(), //enables / displays the y-axis
+		},
+		Series: []chart.Series{
+			chart.ContinuousSeries{
+				Style: chart.Style{
+					Show:        true,
+					StrokeColor: chart.GetDefaultColor(0).WithAlpha(64),
+					FillColor:   chart.GetDefaultColor(0).WithAlpha(64),
+					DotColor:    chart.GetDefaultColor(0),
+					DotWidth:    3.0,
+				},
+				XValues: xValues,
+				YValues: yValues,
+			},
+		},
 	}
-	out += "\n"
 
-	n, err = s.writer.WriteString(string(out) + "\n\n")
+	buffer := bytes.NewBuffer([]byte{})
+	err := graph.Render(chart.PNG, buffer)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal("chart.Render error:", err)
 	}
-	if n == 0 {
+	ioutil.WriteFile("/tmp/resh-graph_"+graphName+".png", buffer.Bytes(), 0644)
-		log.Fatal("Nothing was written", n)
+}
+
+func sortMapByvalue(input map[string]int) []Pair {
+	p := make(PairList, len(input))
+
+	i := 0
+	for k, v := range input {
+		p[i] = Pair{k, v}
+		i++
 	}
+	sort.Sort(sort.Reverse(p))
+	return p
+}
+
+// Pair - A data structure to hold key/value pairs
+type Pair struct {
+	Key   string
+	Value int
 }
+
+// PairList - A slice of pairs that implements sort.Interface to sort by values
+type PairList []Pair
+
+func (p PairList) Len() int           { return len(p) }
+func (p PairList) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }

go.mod

@@ -4,5 +4,9 @@ go 1.12
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/mattn/go-shellwords v1.0.6
+	github.com/wcharczuk/go-chart v2.0.1+incompatible
 	github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa
+	golang.org/x/image v0.0.0-20190902063713-cb417be4ba39 // indirect
 )

go.sum

@@ -1,4 +1,13 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3ZkeUUI=
+github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/wcharczuk/go-chart v2.0.1+incompatible h1:0pz39ZAycJFF7ju/1mepnk26RLVLBCWz1STcD3doU0A=
+github.com/wcharczuk/go-chart v2.0.1+incompatible/go.mod h1:PF5tmL4EIx/7Wf+hEkpCqYi5He4u90sw+0+6FhrryuE=
 github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa h1:rW+Lu6281ed/4XGuVIa4/YebTRNvoUJlfJ44ktEVwZk=
 github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa/go.mod h1:2rx5KE5FLD0HRfkkpyn8JwbVLBdhgeiOb2D2D9LLKM4=
+golang.org/x/image v0.0.0-20190902063713-cb417be4ba39 h1:4dQcAORh9oYBwVSBVIkP489LUPC+f1HBkTYXgmqfR+o=
+golang.org/x/image v0.0.0-20190902063713-cb417be4ba39/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=