Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[client] Back up corrupted state files and present them in the debug bundle #3227

Merged
merged 3 commits into from
Jan 23, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions client/internal/statemanager/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,20 +303,29 @@ func (m *Manager) loadStateFile(deleteCorrupt bool) (map[string]json.RawMessage,

var rawStates map[string]json.RawMessage
if err := json.Unmarshal(data, &rawStates); err != nil {
if deleteCorrupt {
log.Warn("State file appears to be corrupted, attempting to delete it", err)
if err := os.Remove(m.filePath); err != nil {
log.Errorf("Failed to delete corrupted state file: %v", err)
} else {
log.Info("State file deleted")
}
}
m.handleCorruptedState(deleteCorrupt)
return nil, fmt.Errorf("unmarshal states: %w", err)
}

return rawStates, nil
}

// handleCorruptedState creates a backup of a corrupted state file by moving it
func (m *Manager) handleCorruptedState(deleteCorrupt bool) {
if !deleteCorrupt {
return
}
log.Warn("State file appears to be corrupted, attempting to back it up")

backupPath := fmt.Sprintf("%s.corrupted.%d", m.filePath, time.Now().UnixNano())
if err := os.Rename(m.filePath, backupPath); err != nil {
log.Errorf("Failed to backup corrupted state file: %v", err)
return
}

log.Infof("Created backup of corrupted state file at: %s", backupPath)
}

// loadSingleRawState unmarshals a raw state into a concrete state object
func (m *Manager) loadSingleRawState(name string, rawState json.RawMessage) (State, error) {
stateType, ok := m.stateTypes[name]
Expand Down
34 changes: 34 additions & 0 deletions client/server/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,10 @@ func (s *Server) createArchive(bundlePath *os.File, req *proto.DebugBundleReques
log.Errorf("Failed to add state file to debug bundle: %v", err)
}

if err := s.addCorruptedStateFiles(archive); err != nil {
log.Errorf("Failed to add corrupted state files to debug bundle: %v", err)
}

if s.logFile != "console" {
if err := s.addLogfile(req, anonymizer, archive); err != nil {
return fmt.Errorf("add log file: %w", err)
Expand Down Expand Up @@ -403,6 +407,36 @@ func (s *Server) addStateFile(req *proto.DebugBundleRequest, anonymizer *anonymi
return nil
}

func (s *Server) addCorruptedStateFiles(archive *zip.Writer) error {
pattern := statemanager.GetDefaultStatePath()
if pattern == "" {
return nil
}
pattern += "*.corrupted.*"
matches, err := filepath.Glob(pattern)
if err != nil {
return fmt.Errorf("find corrupted state files: %w", err)
}

for _, match := range matches {
data, err := os.ReadFile(match)
if err != nil {
log.Warnf("Failed to read corrupted state file %s: %v", match, err)
continue
}

fileName := filepath.Base(match)
if err := addFileToZip(archive, bytes.NewReader(data), "corrupted_states/"+fileName); err != nil {
log.Warnf("Failed to add corrupted state file %s to zip: %v", fileName, err)
continue
}

log.Debugf("Added corrupted state file to debug bundle: %s", fileName)
}

return nil
}

func (s *Server) addLogfile(req *proto.DebugBundleRequest, anonymizer *anonymize.Anonymizer, archive *zip.Writer) error {
logDir := filepath.Dir(s.logFile)

Expand Down
Loading