Merge pull request #85 from curusarn/bug_handle_invalid_json

skip invalid history records and fix history file during start up

cmd/cli/main.go

@@ -34,6 +34,8 @@ var commit string
 // special constant recognized by RESH wrappers
 const exitCodeExecute = 111
 
+var debug bool
+
 func main() {
 	output, exitCode := runReshCli()
 	fmt.Print(output)
@@ -59,6 +61,7 @@ func runReshCli() (string, int) {
 		log.Fatal("Error reading config:", err)
 	}
 	if config.Debug {
+		debug = true
 		log.SetFlags(log.LstdFlags | log.Lmicroseconds)
 	}
 
@@ -225,20 +228,26 @@ func filterTerms(terms []string) []string {
 }
 
 func newQueryFromString(queryInput string, pwd string) query {
+	if debug {
 		log.Println("QUERY input = <" + queryInput + ">")
+	}
 	terms := strings.Fields(queryInput)
 	var logStr string
 	for _, term := range terms {
 		logStr += " <" + term + ">"
 	}
+	if debug {
 		log.Println("QUERY raw terms =" + logStr)
+	}
 	terms = filterTerms(terms)
 	logStr = ""
 	for _, term := range terms {
 		logStr += " <" + term + ">"
 	}
+	if debug {
 		log.Println("QUERY filtered terms =" + logStr)
 		log.Println("QUERY pwd =" + pwd)
+	}
 	return query{terms: terms, pwd: pwd}
 }
 
@@ -445,9 +454,11 @@ func (m manager) SelectPaste(g *gocui.Gui, v *gocui.View) error {
 }
 
 func (m manager) UpdateData(input string) {
+	if debug {
 		log.Println("EDIT start")
 		log.Println("len(fullRecords) =", len(m.s.fullRecords))
 		log.Println("len(data) =", len(m.s.data))
+	}
 	query := newQueryFromString(input, m.pwd)
 	var data []item
 	itemSet := make(map[string]bool)
@@ -468,7 +479,9 @@ func (m manager) UpdateData(input string) {
 		data = append(data, itm)
 		// log.Println("DATA =", itm.display)
 	}
+	if debug {
 		log.Println("len(tmpdata) =", len(data))
+	}
 	sort.SliceStable(data, func(p, q int) bool {
 		return data[p].hits > data[q].hits
 	})
@@ -480,10 +493,12 @@ func (m manager) UpdateData(input string) {
 		m.s.data = append(m.s.data, itm)
 	}
 	m.s.highlightedItem = 0
+	if debug {
 		log.Println("len(fullRecords) =", len(m.s.fullRecords))
 		log.Println("len(data) =", len(m.s.data))
 		log.Println("EDIT end")
 	}
+}
 
 func (m manager) Edit(v *gocui.View, key gocui.Key, ch rune, mod gocui.Modifier) {
 	gocui.DefaultEditor.Edit(v, key, ch, mod)
@@ -563,8 +578,10 @@ func (m manager) Layout(g *gocui.Gui) error {
 		// 	v.SetHighlight(m.s.highlightedItem, true)
 		// }
 	}
+	if debug {
 		log.Println("len(data) =", len(m.s.data))
 		log.Println("highlightedItem =", m.s.highlightedItem)
+	}
 	return nil
 }
 

pkg/histfile/histfile.go

@@ -46,13 +46,11 @@ func New(input chan records.Record, sessionsToDrop chan string,
 	go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB)
 	go hf.writer(input, signals, shutdownDone)
 	go hf.sessionGC(sessionsToDrop)
-	go hf.loadFullRecords()
 	return &hf
 }
 
 // load records from resh history, reverse, enrich and save
-func (h *Histfile) loadFullRecords() {
+func (h *Histfile) loadFullRecords(recs []records.Record) {
-	recs := records.LoadFromFile(h.historyPath, math.MaxInt32)
 	for i := len(recs) - 1; i >= 0; i-- {
 		rec := recs[i]
 		h.fullRecords.AddRecord(rec)
@@ -83,10 +81,11 @@ func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHi
 		maxInitHistSize = math.MaxInt32
 	}
 	log.Println("histfile: Loading resh history from file ...")
-	reshCmdLines := histlist.New()
+	history := records.LoadFromFile(h.historyPath, math.MaxInt32)
+	go h.loadFullRecords(history)
 	// NOTE: keeping this weird interface for now because we might use it in the future
 	//			when we only load bash or zsh history
-	records.LoadCmdLinesFromFile(&reshCmdLines, h.historyPath, maxInitHistSize)
+	reshCmdLines := loadCmdLines(history)
 	log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List))
 	if useNativeHistories == false {
 		h.bashCmdLines = reshCmdLines
@@ -230,3 +229,26 @@ func (h *Histfile) DumpRecords() histcli.Histcli {
 	// don't forget locks in the future
 	return h.fullRecords
 }
+
+func loadCmdLines(recs []records.Record) histlist.Histlist {
+	hl := histlist.New()
+	// go from bottom and deduplicate
+	var cmdLines []string
+	cmdLinesSet := map[string]bool{}
+	for i := len(recs) - 1; i >= 0; i-- {
+		cmdLine := recs[i].CmdLine
+		if cmdLinesSet[cmdLine] {
+			continue
+		}
+		cmdLinesSet[cmdLine] = true
+		cmdLines = append([]string{cmdLine}, cmdLines...)
+		// if len(cmdLines) > limit {
+		// 	break
+		// }
+	}
+	// add everything to histlist
+	for _, cmdLine := range cmdLines {
+		hl.AddCmdLine(cmdLine)
+	}
+	return hl
+}

pkg/records/records.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"encoding/json"
 	"errors"
+	"io"
 	"log"
 	"math"
 	"os"
@@ -173,16 +174,16 @@ func Enriched(r Record) EnrichedRecord {
 	record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine)
 	if err != nil {
 		record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error())
-		rec, _ := record.ToString()
+		// rec, _ := record.ToString()
-		log.Println("Invalid command:", rec)
+		// log.Println("Invalid command:", rec)
 		record.Invalid = true
 		return record
 	}
 	err = r.Validate()
 	if err != nil {
 		record.Errors = append(record.Errors, "Validate error:"+err.Error())
-		rec, _ := record.ToString()
+		// rec, _ := record.ToString()
-		log.Println("Invalid command:", rec)
+		// log.Println("Invalid command:", rec)
 		record.Invalid = true
 	}
 	return record
@@ -306,7 +307,7 @@ func Stripped(r EnrichedRecord) EnrichedRecord {
 func GetCommandAndFirstWord(cmdLine string) (string, string, error) {
 	args, err := shellwords.Parse(cmdLine)
 	if err != nil {
-		log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
+		// log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
 		return "", "", err
 	}
 	if len(args) == 0 {
@@ -456,31 +457,10 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 {
 	return dist
 }
 
-// LoadCmdLinesFromFile loads cmdlines from file
-func LoadCmdLinesFromFile(hl *histlist.Histlist, fname string, limit int) {
-	recs := LoadFromFile(fname, limit*3) // assume that at least 1/3 of commands is unique
-	// go from bottom and deduplicate
-	var cmdLines []string
-	cmdLinesSet := map[string]bool{}
-	for i := len(recs) - 1; i >= 0; i-- {
-		cmdLine := recs[i].CmdLine
-		if cmdLinesSet[cmdLine] {
-			continue
-		}
-		cmdLinesSet[cmdLine] = true
-		cmdLines = append([]string{cmdLine}, cmdLines...)
-		if len(cmdLines) > limit {
-			break
-		}
-	}
-	// add everything to histlist
-	for _, cmdLine := range cmdLines {
-		hl.AddCmdLine(cmdLine)
-	}
-}
-
 // LoadFromFile loads records from 'fname' file
 func LoadFromFile(fname string, limit int) []Record {
+	const allowedErrors = 1
+	var encounteredErrors int
 	// NOTE: limit does nothing atm
 	var recs []Record
 	file, err := os.Open(fname)
@@ -492,7 +472,10 @@ func LoadFromFile(fname string, limit int) []Record {
 	defer file.Close()
 
 	scanner := bufio.NewScanner(file)
+	var i int
+	var firstErrLine int
 	for scanner.Scan() {
+		i++
 		record := Record{}
 		fallbackRecord := FallbackRecord{}
 		line := scanner.Text()
@@ -500,16 +483,78 @@ func LoadFromFile(fname string, limit int) []Record {
 		if err != nil {
 			err = json.Unmarshal([]byte(line), &fallbackRecord)
 			if err != nil {
+				if encounteredErrors == 0 {
+					firstErrLine = i
+				}
+				encounteredErrors++
 				log.Println("Line:", line)
-				log.Fatal("Decoding error:", err)
+				log.Println("Decoding error:", err)
+				if encounteredErrors > allowedErrors {
+					log.Fatalf("Fatal: Encountered more than %d decoding errors (%d)", allowedErrors, encounteredErrors)
+				}
 			}
 			record = Convert(&fallbackRecord)
 		}
 		recs = append(recs, record)
 	}
+	if encounteredErrors > 0 {
+		// fix errors in the history file
+		log.Printf("There were %d decoding errors, the first error happend on line %d/%d", encounteredErrors, firstErrLine, i)
+		log.Println("Backing up current history file ...")
+		err := copyFile(fname, fname+".bak")
+		if err != nil {
+			log.Fatalln("Failed to backup history file with decode errors")
+		}
+		log.Println("Writing out a history file without errors ...")
+		err = writeHistory(fname, recs)
+		if err != nil {
+			log.Fatalln("Fatal: Failed write out new history")
+		}
+	}
 	return recs
 }
 
+func copyFile(source, dest string) error {
+	from, err := os.Open(source)
+	if err != nil {
+		// log.Println("Open() resh history file error:", err)
+		return err
+	}
+	defer from.Close()
+
+	// to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666)
+	to, err := os.Create(dest)
+	if err != nil {
+		// log.Println("Create() resh history backup error:", err)
+		return err
+	}
+	defer to.Close()
+
+	_, err = io.Copy(to, from)
+	if err != nil {
+		// log.Println("Copy() resh history to backup error:", err)
+		return err
+	}
+	return nil
+}
+
+func writeHistory(fname string, history []Record) error {
+	file, err := os.Create(fname)
+	if err != nil {
+		// log.Println("Create() resh history error:", err)
+		return err
+	}
+	defer file.Close()
+	for _, rec := range history {
+		jsn, err := json.Marshal(rec)
+		if err != nil {
+			log.Fatalln("Encode error!")
+		}
+		file.Write(append(jsn, []byte("\n")...))
+	}
+	return nil
+}
+
 // LoadCmdLinesFromZshFile loads cmdlines from zsh history file
 func LoadCmdLinesFromZshFile(fname string) histlist.Histlist {
 	hl := histlist.New()