Fix broken snapshots via restic repair snapshots --forget (#21907)

Co-authored-by: Michael Eischer <michael.eischer@fau.de>

cmd/restic/cmd_check.go

@@ -339,6 +339,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
 
 	printer.P("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
+	var brokenSnapshots []string
 	var wg sync.WaitGroup
 
 	wg.Add(1)
@@ -351,13 +352,17 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
 
 	for err := range errChan {
 		errorsFound = true
-		if e, ok := err.(*checker.TreeError); ok {
+		switch e := err.(type) {
+		case *checker.TreeError:
 			printer.E("error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
 				summary.NumErrors++
 				printer.E("  %v\n", treeErr)
 			}
-		} else {
+		case *checker.SnapshotError:
+			printer.E("snapshot error %v: %v", e.ID, e.Message)
+			brokenSnapshots = append(brokenSnapshots, e.ID)
+		default:
 			summary.NumErrors++
 			printer.E("error: %v\n", err)
 		}
@@ -412,12 +417,18 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
 		printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
 	}
 
+	if len(brokenSnapshots) > 0 {
+		printer.E("\nThe repository contains damaged snapshot files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
+		printer.E("restic repair snapshots --forget %s\n\n", strings.Join(brokenSnapshots, " "))
+		printer.E("Damaged snapshot files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
+	}
+
 	if ctx.Err() != nil {
 		return summary, ctx.Err()
 	}
 
 	if errorsFound {
-		if len(salvagePacks) == 0 {
+		if len(salvagePacks) == 0 && len(brokenSnapshots) == 0 {
 			printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n")
 		}
 		return summary, errors.Fatal("repository contains errors")

cmd/restic/cmd_repair_snapshots.go

@@ -78,6 +78,37 @@ func (opts *RepairOptions) AddFlags(f *pflag.FlagSet) {
 	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
 }
 
+// handleUnreadableSnapshotFile is called when FindAll returns an error for ID 'id'
+func handleUnreadableSnapshotFile(
+	ctx context.Context,
+	be restic.Lister,
+	repo restic.Repository,
+	opts RepairOptions,
+	id string,
+	args []string,
+	printer restic.Printer,
+) (bool, error) {
+	brokenID, err := restic.Find(ctx, be, restic.SnapshotFile, id)
+	if err != nil {
+		return false, err
+	}
+
+	if opts.Forget && slices.Index(args, id) >= 0 {
+		if opts.DryRun {
+			printer.P("would remove unreadable snapshot %v", brokenID)
+			return true, nil
+		}
+
+		if err := repo.RemoveUnpacked(ctx, restic.WriteableSnapshotFile, brokenID); err != nil {
+			return false, errors.Wrapf(err, "unable to remove unreadable snapshot file %v", brokenID)
+		}
+		printer.P("removed unreadable snapshot %v", brokenID)
+		return true, nil
+	}
+
+	return false, errors.Fatalf("snapshot file %[1]s is unreadable, use `restic repair snapshots --forget %[1]s` to remove it. Original error: %v", brokenID, err)
+}
+
 func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOptions, args []string, term ui.Terminal) error {
 	printer := progress.NewTerminalPrinter(false, gopts.Verbosity, term)
 
@@ -96,10 +127,11 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
 		return err
 	}
 
-	// Three error cases are checked:
+	// Four error cases are checked:
 	// - tree is a nil tree (-> will be replaced by an empty tree)
 	// - trees which cannot be loaded (-> the tree contents will be removed)
 	// - files whose contents are not fully available  (-> file will be modified)
+	// - *checker.SnapshotError
 	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
 		RewriteNode: func(node *data.Node, path string) *data.Node {
 			if node.Type == data.NodeTypeIrregular || node.Type == data.NodeTypeInvalid {
@@ -146,7 +178,15 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
 	})
 
 	changedCount := 0
-	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args, printer) {
+	errOuter := opts.SnapshotFilter.FindAll(ctx, snapshotLister, repo, args, func(id string, sn *data.Snapshot, err error) error {
+		if err != nil {
+			changed, err := handleUnreadableSnapshotFile(ctx, snapshotLister, repo, opts, id, args, printer)
+			if changed {
+				changedCount++
+			}
+			return err
+		}
+
 		printer.P("\n%v", sn)
 		changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
 			func(ctx context.Context, sn *data.Snapshot, uploader restic.BlobSaver) (restic.ID, *data.SnapshotSummary, error) {
@@ -159,9 +199,11 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
 		if changed {
 			changedCount++
 		}
-	}
-	if ctx.Err() != nil {
-		return ctx.Err()
+		return nil
+	})
+
+	if errOuter != nil {
+		return errOuter
 	}
 
 	printer.P("")

cmd/restic/cmd_repair_snapshots_integration_test.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"hash/fnv"
 	"io"
 	"math/rand"
@@ -136,3 +138,38 @@ func TestRepairSnapshotsIntact(t *testing.T) {
 	rtest.Assert(t, reflect.DeepEqual(oldSnapshotIDs, snapshotIDs), "unexpected snapshot id mismatch %v vs. %v", oldSnapshotIDs, snapshotIDs)
 	testRunCheck(t, env.gopts)
 }
+
+func TestRepairSnapshotsBrokenSnapshots(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	// create backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
+
+	// create zero length file in snapshots/
+	// will fail with
+	// failed to load snapshot 1d204771: LoadRaw(<snapshot/1d20477115>): invalid data returned
+	handle, err := os.Create(filepath.Join(env.repo, "snapshots", "1d20477115fb872069a28a80ffb95a82cb8b1b1920de046a68c0195da63f30cf"))
+	rtest.OK(t, err)
+	rtest.OK(t, handle.Close())
+
+	// create some file with a correct sha256 name in snapshots/, will fail with
+	// failed to load snapshot abcd1234: ciphertext verification failed
+	contents := rtest.Random(1234567890123, 42)
+	sha256Contents := sha256.Sum256(contents)
+	target := hex.EncodeToString(sha256Contents[:])
+	rtest.OK(t, os.WriteFile(filepath.Join(env.repo, "snapshots", target), contents, 0o600))
+
+	// run repair snapshots
+	repairOpts := RepairOptions{Forget: true}
+	env.gopts.BackendTestHook = nil
+	_, err = withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
+		return runRepairSnapshots(ctx, gopts, repairOpts, []string{"1d204771", target[:8]}, gopts.Term)
+	})
+	rtest.OK(t, err)
+
+	// verify that there are no snapshot errors
+	testRunCheck(t, env.gopts)
+}