Mirror
/
resh
mirror of https://github.com/curusarn/resh
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #85 from curusarn/bug_handle_invalid_json

Browse Source 
skip invalid history records and fix history file during start up
pull/89/head v2.5.15
Šimon Let 6 years ago committed by GitHub
parent 96b5e9d45f 27db75157b
commit d120929030
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 131 additions and 47 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 17
      cmd/cli/main.go
  2. 32
      pkg/histfile/histfile.go
  3. 103
      pkg/records/records.go

17
cmd/cli/main.go
Unescape View File

@ -34,6 +34,8 @@ var commit string
// special constant recognized by RESH wrappers
const exitCodeExecute = 111
var debug bool
func main() {
output, exitCode := runReshCli()
fmt.Print(output)
@ -59,6 +61,7 @@ func runReshCli() (string, int) {
log.Fatal("Error reading config:", err)
}
if config.Debug {
debug = true
log.SetFlags(log.LstdFlags | log.Lmicroseconds)
}
@ -225,20 +228,26 @@ func filterTerms(terms []string) []string {
}
func newQueryFromString(queryInput string, pwd string) query {
if debug {
log.Println("QUERY input = <" + queryInput + ">")
}
terms := strings.Fields(queryInput)
var logStr string
for _, term := range terms {
logStr += " <" + term + ">"
}
if debug {
log.Println("QUERY raw terms =" + logStr)
}
terms = filterTerms(terms)
logStr = ""
for _, term := range terms {
logStr += " <" + term + ">"
}
if debug {
log.Println("QUERY filtered terms =" + logStr)
log.Println("QUERY pwd =" + pwd)
}
return query{terms: terms, pwd: pwd}
}
@ -445,9 +454,11 @@ func (m manager) SelectPaste(g *gocui.Gui, v *gocui.View) error {
}
func (m manager) UpdateData(input string) {
if debug {
log.Println("EDIT start")
log.Println("len(fullRecords) =", len(m.s.fullRecords))
log.Println("len(data) =", len(m.s.data))
}
query := newQueryFromString(input, m.pwd)
var data []item
itemSet := make(map[string]bool)
@ -468,7 +479,9 @@ func (m manager) UpdateData(input string) {
data = append(data, itm)
// log.Println("DATA =", itm.display)
}
if debug {
log.Println("len(tmpdata) =", len(data))
}
sort.SliceStable(data, func(p, q int) bool {
return data[p].hits > data[q].hits
})
@ -480,10 +493,12 @@ func (m manager) UpdateData(input string) {
m.s.data = append(m.s.data, itm)
}
m.s.highlightedItem = 0
if debug {
log.Println("len(fullRecords) =", len(m.s.fullRecords))
log.Println("len(data) =", len(m.s.data))
log.Println("EDIT end")
}
}
func (m manager) Edit(v *gocui.View, key gocui.Key, ch rune, mod gocui.Modifier) {
gocui.DefaultEditor.Edit(v, key, ch, mod)
@ -563,8 +578,10 @@ func (m manager) Layout(g *gocui.Gui) error {
// v.SetHighlight(m.s.highlightedItem, true)
// }
}
if debug {
log.Println("len(data) =", len(m.s.data))
log.Println("highlightedItem =", m.s.highlightedItem)
}
return nil
}

32
pkg/histfile/histfile.go
Unescape View File

@ -46,13 +46,11 @@ func New(input chan records.Record, sessionsToDrop chan string,
go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB)
go hf.writer(input, signals, shutdownDone)
go hf.sessionGC(sessionsToDrop)
go hf.loadFullRecords()
return &hf
}
// load records from resh history, reverse, enrich and save
func (h *Histfile) loadFullRecords() {
recs := records.LoadFromFile(h.historyPath, math.MaxInt32)
func (h *Histfile) loadFullRecords(recs []records.Record) {
for i := len(recs) - 1; i >= 0; i-- {
rec := recs[i]
h.fullRecords.AddRecord(rec)
@ -83,10 +81,11 @@ func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHi
maxInitHistSize = math.MaxInt32
}
log.Println("histfile: Loading resh history from file ...")
reshCmdLines := histlist.New()
history := records.LoadFromFile(h.historyPath, math.MaxInt32)
go h.loadFullRecords(history)
// NOTE: keeping this weird interface for now because we might use it in the future
// when we only load bash or zsh history
records.LoadCmdLinesFromFile(&reshCmdLines, h.historyPath, maxInitHistSize)
reshCmdLines := loadCmdLines(history)
log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List))
if useNativeHistories == false {
h.bashCmdLines = reshCmdLines
@ -230,3 +229,26 @@ func (h *Histfile) DumpRecords() histcli.Histcli {
// don't forget locks in the future
return h.fullRecords
}
func loadCmdLines(recs []records.Record) histlist.Histlist {
hl := histlist.New()
// go from bottom and deduplicate
var cmdLines []string
cmdLinesSet := map[string]bool{}
for i := len(recs) - 1; i >= 0; i-- {
cmdLine := recs[i].CmdLine
if cmdLinesSet[cmdLine] {
continue
}
cmdLinesSet[cmdLine] = true
cmdLines = append([]string{cmdLine}, cmdLines...)
// if len(cmdLines) > limit {
// break
// }
}
// add everything to histlist
for _, cmdLine := range cmdLines {
hl.AddCmdLine(cmdLine)
}
return hl
}

103
pkg/records/records.go
Unescape View File

@ -4,6 +4,7 @@ import (
"bufio"
"encoding/json"
"errors"
"io"
"log"
"math"
"os"
@ -173,16 +174,16 @@ func Enriched(r Record) EnrichedRecord {
record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine)
if err != nil {
record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error())
rec, _ := record.ToString()
log.Println("Invalid command:", rec)
// rec, _ := record.ToString()
// log.Println("Invalid command:", rec)
record.Invalid = true
return record
}
err = r.Validate()
if err != nil {
record.Errors = append(record.Errors, "Validate error:"+err.Error())
rec, _ := record.ToString()
log.Println("Invalid command:", rec)
// rec, _ := record.ToString()
// log.Println("Invalid command:", rec)
record.Invalid = true
}
return record
@ -306,7 +307,7 @@ func Stripped(r EnrichedRecord) EnrichedRecord {
func GetCommandAndFirstWord(cmdLine string) (string, string, error) {
args, err := shellwords.Parse(cmdLine)
if err != nil {
log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
// log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
return "", "", err
}
if len(args) == 0 {
@ -456,31 +457,10 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 {
return dist
}
// LoadCmdLinesFromFile loads cmdlines from file
func LoadCmdLinesFromFile(hl *histlist.Histlist, fname string, limit int) {
recs := LoadFromFile(fname, limit*3) // assume that at least 1/3 of commands is unique
// go from bottom and deduplicate
var cmdLines []string
cmdLinesSet := map[string]bool{}
for i := len(recs) - 1; i >= 0; i-- {
cmdLine := recs[i].CmdLine
if cmdLinesSet[cmdLine] {
continue
}
cmdLinesSet[cmdLine] = true
cmdLines = append([]string{cmdLine}, cmdLines...)
if len(cmdLines) > limit {
break
}
}
// add everything to histlist
for _, cmdLine := range cmdLines {
hl.AddCmdLine(cmdLine)
}
}
// LoadFromFile loads records from 'fname' file
func LoadFromFile(fname string, limit int) []Record {
const allowedErrors = 1
var encounteredErrors int
// NOTE: limit does nothing atm
var recs []Record
file, err := os.Open(fname)
@ -492,7 +472,10 @@ func LoadFromFile(fname string, limit int) []Record {
defer file.Close()
scanner := bufio.NewScanner(file)
var i int
var firstErrLine int
for scanner.Scan() {
i++
record := Record{}
fallbackRecord := FallbackRecord{}
line := scanner.Text()
@ -500,16 +483,78 @@ func LoadFromFile(fname string, limit int) []Record {
if err != nil {
err = json.Unmarshal([]byte(line), &fallbackRecord)
if err != nil {
if encounteredErrors == 0 {
firstErrLine = i
}
encounteredErrors++
log.Println("Line:", line)
log.Fatal("Decoding error:", err)
log.Println("Decoding error:", err)
if encounteredErrors > allowedErrors {
log.Fatalf("Fatal: Encountered more than %d decoding errors (%d)", allowedErrors, encounteredErrors)
}
}
record = Convert(&fallbackRecord)
}
recs = append(recs, record)
}
if encounteredErrors > 0 {
// fix errors in the history file
log.Printf("There were %d decoding errors, the first error happend on line %d/%d", encounteredErrors, firstErrLine, i)
log.Println("Backing up current history file ...")
err := copyFile(fname, fname+".bak")
if err != nil {
log.Fatalln("Failed to backup history file with decode errors")
}
log.Println("Writing out a history file without errors ...")
err = writeHistory(fname, recs)
if err != nil {
log.Fatalln("Fatal: Failed write out new history")
}
}
return recs
}
func copyFile(source, dest string) error {
from, err := os.Open(source)
if err != nil {
// log.Println("Open() resh history file error:", err)
return err
}
defer from.Close()
// to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666)
to, err := os.Create(dest)
if err != nil {
// log.Println("Create() resh history backup error:", err)
return err
}
defer to.Close()
_, err = io.Copy(to, from)
if err != nil {
// log.Println("Copy() resh history to backup error:", err)
return err
}
return nil
}
func writeHistory(fname string, history []Record) error {
file, err := os.Create(fname)
if err != nil {
// log.Println("Create() resh history error:", err)
return err
}
defer file.Close()
for _, rec := range history {
jsn, err := json.Marshal(rec)
if err != nil {
log.Fatalln("Encode error!")
}
file.Write(append(jsn, []byte("\n")...))
}
return nil
}
// LoadCmdLinesFromZshFile loads cmdlines from zsh history file
func LoadCmdLinesFromZshFile(fname string) histlist.Histlist {
hl := histlist.New()

Write Preview
Loading…
Cancel
Save