add fallback record for old resh history records, minor changes

common/resh-common.go

@@ -1,5 +1,8 @@
 package common
 
+import "strconv"
+
+// Record representing single executed command with its metadata
 type Record struct {
 	// core
 	CmdLine   string `json:"cmdLine"`
@@ -65,6 +68,141 @@ type Record struct {
 	CmdLength int `json:"cmdLength"`
 }
 
+// FallbackRecord when record is too old and can't be parsed into regular Record
+type FallbackRecord struct {
+	// older version of the record where cols and lines are int
+
+	// core
+	CmdLine   string `json:"cmdLine"`
+	ExitCode  int    `json:"exitCode"`
+	Shell     string `json:"shell"`
+	Uname     string `json:"uname"`
+	SessionId string `json:"sessionId"`
+
+	// posix
+	Cols  int    `json:"cols"`  // notice the in type
+	Lines int    `json:"lines"` // notice the in type
+	Home  string `json:"home"`
+	Lang  string `json:"lang"`
+	LcAll string `json:"lcAll"`
+	Login string `json:"login"`
+	//Path     string `json:"path"`
+	Pwd      string `json:"pwd"`
+	PwdAfter string `json:"pwdAfter"`
+	ShellEnv string `json:"shellEnv"`
+	Term     string `json:"term"`
+
+	// non-posix"`
+	RealPwd      string `json:"realPwd"`
+	RealPwdAfter string `json:"realPwdAfter"`
+	Pid          int    `json:"pid"`
+	SessionPid   int    `json:"sessionPid"`
+	Host         string `json:"host"`
+	Hosttype     string `json:"hosttype"`
+	Ostype       string `json:"ostype"`
+	Machtype     string `json:"machtype"`
+	Shlvl        int    `json:"shlvl"`
+
+	// before after
+	TimezoneBefore string `json:"timezoneBefore"`
+	TimezoneAfter  string `json:"timezoneAfter"`
+
+	RealtimeBefore      float64 `json:"realtimeBefore"`
+	RealtimeAfter       float64 `json:"realtimeAfter"`
+	RealtimeBeforeLocal float64 `json:"realtimeBeforeLocal"`
+	RealtimeAfterLocal  float64 `json:"realtimeAfterLocal"`
+
+	RealtimeDuration          float64 `json:"realtimeDuration"`
+	RealtimeSinceSessionStart float64 `json:"realtimeSinceSessionStart"`
+	RealtimeSinceBoot         float64 `json:"realtimeSinceBoot"`
+	//Logs []string      `json: "logs"`
+
+	GitDir          string `json:"gitDir"`
+	GitRealDir      string `json:"gitRealDir"`
+	GitOriginRemote string `json:"gitOriginRemote"`
+	MachineId       string `json:"machineId"`
+
+	OsReleaseId         string `json:"osReleaseId"`
+	OsReleaseVersionId  string `json:"osReleaseVersionId"`
+	OsReleaseIdLike     string `json:"osReleaseIdLike"`
+	OsReleaseName       string `json:"osReleaseName"`
+	OsReleasePrettyName string `json:"osReleasePrettyName"`
+
+	ReshUuid     string `json:"reshUuid"`
+	ReshVersion  string `json:"reshVersion"`
+	ReshRevision string `json:"reshRevision"`
+
+	// added by sanitizatizer
+	CmdLength int `json:"cmdLength"`
+}
+
+// ConvertRecord from FallbackRecord to Record
+func ConvertRecord(r *FallbackRecord) Record {
+	return Record{
+		// core
+		CmdLine:   r.CmdLine,
+		ExitCode:  r.ExitCode,
+		Shell:     r.Shell,
+		Uname:     r.Uname,
+		SessionId: r.SessionId,
+
+		// posix
+		// these two lines are the only reason we are doing this
+		Cols:  strconv.Itoa(r.Cols),
+		Lines: strconv.Itoa(r.Lines),
+
+		Home:  r.Home,
+		Lang:  r.Lang,
+		LcAll: r.LcAll,
+		Login: r.Login,
+		// Path:     r.path,
+		Pwd:      r.Pwd,
+		PwdAfter: r.PwdAfter,
+		ShellEnv: r.ShellEnv,
+		Term:     r.Term,
+
+		// non-posix
+		RealPwd:      r.RealPwd,
+		RealPwdAfter: r.RealPwdAfter,
+		Pid:          r.Pid,
+		SessionPid:   r.SessionPid,
+		Host:         r.Host,
+		Hosttype:     r.Hosttype,
+		Ostype:       r.Ostype,
+		Machtype:     r.Machtype,
+		Shlvl:        r.Shlvl,
+
+		// before after
+		TimezoneBefore: r.TimezoneBefore,
+		TimezoneAfter:  r.TimezoneAfter,
+
+		RealtimeBefore:      r.RealtimeBefore,
+		RealtimeAfter:       r.RealtimeAfter,
+		RealtimeBeforeLocal: r.RealtimeBeforeLocal,
+		RealtimeAfterLocal:  r.RealtimeAfterLocal,
+
+		RealtimeDuration:          r.RealtimeDuration,
+		RealtimeSinceSessionStart: r.RealtimeSinceSessionStart,
+		RealtimeSinceBoot:         r.RealtimeSinceBoot,
+
+		GitDir:          r.GitDir,
+		GitRealDir:      r.GitRealDir,
+		GitOriginRemote: r.GitOriginRemote,
+		MachineId:       r.MachineId,
+
+		OsReleaseId:         r.OsReleaseId,
+		OsReleaseVersionId:  r.OsReleaseVersionId,
+		OsReleaseIdLike:     r.OsReleaseIdLike,
+		OsReleaseName:       r.OsReleaseName,
+		OsReleasePrettyName: r.OsReleasePrettyName,
+
+		ReshUuid:     r.ReshUuid,
+		ReshVersion:  r.ReshVersion,
+		ReshRevision: r.ReshRevision,
+	}
+}
+
+// Config struct
 type Config struct {
 	Port int
 }

sanitize-history/resh-sanitize-history.go

@@ -78,24 +78,26 @@ func main() {
 	scanner := bufio.NewScanner(inputFile)
 	for scanner.Scan() {
 		record := common.Record{}
+		fallbackRecord := common.FallbackRecord{}
 		line := scanner.Text()
 		err = json.Unmarshal([]byte(line), &record)
 		if err != nil {
-			log.Println("Decoding error:", err)
-			log.Println("Line:", line)
-			return
+			err = json.Unmarshal([]byte(line), &fallbackRecord)
+			if err != nil {
+				log.Println("Line:", line)
+				log.Fatal("Decoding error:", err)
+			}
+			record = common.ConvertRecord(&fallbackRecord)
 		}
 		err = sanitizer.sanitizeRecord(&record)
 		if err != nil {
-			log.Println("Sanitization error:", err)
 			log.Println("Line:", line)
-			return
+			log.Fatal("Sanitization error:", err)
 		}
 		outLine, err := json.Marshal(&record)
 		if err != nil {
-			log.Println("Encoding error:", err)
 			log.Println("Line:", line)
-			return
+			log.Fatal("Encoding error:", err)
 		}
 		if useStdout {
 			fmt.Println(string(outLine))
@@ -220,7 +222,7 @@ func (s *sanitizer) sanitizeCmdLine(cmdLine string) (string, error) {
 		case false:
 			// split command on all non-letter and non-digit characters
 			if unicode.IsLetter(r) == false && unicode.IsDigit(r) == false {
-				// TODO: decide if we want to split on tokens
+				// TODO: decide if we want to split on "-" and "_"
 				// split token
 				if len(buff) > 0 {
 					sanToken, err := s.sanitizeCmdToken(buff)