From cc546b71e3469788bb5a85345848b1a47ea2d3e6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 May 2026 22:48:38 +0200 Subject: [PATCH] debug: move code requring internal access to repository package --- cmd/restic/cmd_debug.go | 327 +-------------------------------- internal/repository/debug.go | 338 +++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 318 deletions(-) create mode 100644 internal/repository/debug.go diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 2d3a8d8d3..65c25b79e 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -4,28 +4,18 @@ package main import ( "context" - "crypto/aes" - "crypto/cipher" "encoding/json" "fmt" "io" - "os" - "runtime" "sync" - "time" - "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" "github.com/spf13/pflag" - "golang.org/x/sync/errgroup" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/global" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/repository/index" - "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/progress" @@ -172,17 +162,6 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, }) } -func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error { - return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error { - printer.S("index_id: %v", id) - if err != nil { - return err - } - - return idx.Dump(wr) - }) -} - func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error { printer := ui.NewProgressPrinter(false, gopts.Verbosity, term) @@ -200,7 +179,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term switch tpe { case "indexes": - return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) + return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) case "snapshots": return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter()) case "packs": @@ -213,7 +192,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term } printer.S("indexes:") - err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) + err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) if err != nil { return err } @@ -224,225 +203,6 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term } } -func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte { - if bytewise { - printer.S(" trying to repair blob by finding a broken byte") - } else { - printer.S(" trying to repair blob with single bit flip") - } - - ch := make(chan int) - var wg errgroup.Group - done := make(chan struct{}) - var fixed []byte - var found bool - - workers := runtime.GOMAXPROCS(0) - printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0)) - for i := 0; i < workers; i++ { - wg.Go(func() error { - // make a local copy of the buffer - buf := make([]byte, len(input)) - copy(buf, input) - - testFlip := func(idx int, pattern byte) bool { - // flip bits - buf[idx] ^= pattern - - nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) - if err == nil { - printer.S("") - printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern) - printer.S(" hash is %v", restic.Hash(plaintext)) - close(done) - found = true - fixed = plaintext - return true - } - - // flip bits back - buf[idx] ^= pattern - return false - } - - for i := range ch { - if bytewise { - for j := 0; j < 255; j++ { - if testFlip(i, byte(j)) { - return nil - } - } - } else { - for j := 0; j < 7; j++ { - // flip each bit once - if testFlip(i, (1 << uint(j))) { - return nil - } - } - } - } - return nil - }) - } - - wg.Go(func() error { - defer close(ch) - - start := time.Now() - info := time.Now() - for i := range input { - select { - case ch <- i: - case <-done: - printer.S(" done after %v", time.Since(start)) - return nil - } - - if time.Since(info) > time.Second { - secs := time.Since(start).Seconds() - gps := float64(i) / secs - remaining := len(input) - i - eta := time.Duration(float64(remaining)/gps) * time.Second - - printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", - i, len(input), float32(i)/float32(len(input))*100, gps, eta) - info = time.Now() - } - } - return nil - }) - err := wg.Wait() - if err != nil { - panic("all go routines can only return nil") - } - - if !found { - printer.S("\n blob could not be repaired") - } - return fixed -} - -func decryptUnsigned(k *crypto.Key, buf []byte) []byte { - // strip signature at the end - l := len(buf) - nonce, ct := buf[:16], buf[16:l-16] - out := make([]byte, len(ct)) - - c, err := aes.NewCipher(k.EncryptionKey[:]) - if err != nil { - panic(fmt.Sprintf("unable to create cipher: %v", err)) - } - e := cipher.NewCTR(c, nonce) - e.XORKeyStream(out, ct) - - return out -} - -func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { - dec, err := zstd.NewReader(nil) - if err != nil { - panic(err) - } - - pack, err := repo.LoadRaw(ctx, restic.PackFile, packID) - // allow processing broken pack files - if pack == nil { - return err - } - - err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { - for _, blob := range list { - printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length) - if int(blob.Offset+blob.Length) > len(pack) { - printer.E("skipping truncated blob") - continue - } - buf := pack[blob.Offset : blob.Offset+blob.Length] - key := repo.Key() - - nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) - outputPrefix := "" - filePrefix := "" - if err != nil { - printer.E("error decrypting blob: %v", err) - if opts.TryRepair || opts.RepairByte { - plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer) - } - if plaintext != nil { - outputPrefix = "repaired " - filePrefix = "repaired-" - } else { - plaintext = decryptUnsigned(key, buf) - err = storePlainBlob(blob.ID, "damaged-", plaintext, printer) - if err != nil { - return err - } - continue - } - } - - if blob.IsCompressed() { - decompressed, err := dec.DecodeAll(plaintext, nil) - if err != nil { - printer.S(" failed to decompress blob %v", blob.ID) - } - if decompressed != nil { - plaintext = decompressed - } - } - - id := restic.Hash(plaintext) - var prefix string - if !id.Equal(blob.ID) { - printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID) - prefix = "wrong-hash-" - } else { - printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id) - prefix = "correct-" - } - if opts.ExtractPack { - err = storePlainBlob(id, filePrefix+prefix, plaintext, printer) - if err != nil { - return err - } - } - if opts.ReuploadBlobs { - _, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true) - if err != nil { - return err - } - printer.S(" uploaded %v %v", blob.Type, id) - } - } - return nil - }) - return err -} - -func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error { - filename := fmt.Sprintf("%s%s.bin", prefix, id) - f, err := os.Create(filename) - if err != nil { - return err - } - - _, err = f.Write(plain) - if err != nil { - _ = f.Close() - return err - } - - err = f.Close() - if err != nil { - return err - } - - printer.S("decrypt of blob %v stored at %v", id, filename) - return nil -} - func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error { printer := ui.NewProgressPrinter(false, gopts.Verbosity, term) @@ -478,8 +238,14 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin return err } + examineOpts := repository.ExaminePackOptions{ + TryRepair: opts.TryRepair, + RepairByte: opts.RepairByte, + ExtractPack: opts.ExtractPack, + ReuploadBlobs: opts.ReuploadBlobs, + } for _, id := range ids { - err := examinePack(ctx, opts, repo, id, printer) + err := repository.ExaminePack(ctx, repo, id, examineOpts, printer) if err != nil { printer.E("error: %v", err) } @@ -489,78 +255,3 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin } return nil } - -func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error { - printer.S("examine %v", id) - - buf, err := repo.LoadRaw(ctx, restic.PackFile, id) - // also process damaged pack files - if buf == nil { - return err - } - printer.S(" file size is %v", len(buf)) - gotID := restic.Hash(buf) - if !id.Equal(gotID) { - printer.S(" wanted hash %v, got %v", id, gotID) - } else { - printer.S(" hash for file content matches") - } - - printer.S(" ========================================") - printer.S(" looking for info in the indexes") - - blobsLoaded := false - // examine all data the indexes have for the pack file - for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { - blobs := b.Blobs - if len(blobs) == 0 { - continue - } - - checkPackSize(blobs, len(buf), printer) - - err = loadBlobs(ctx, opts, repo, id, blobs, printer) - if err != nil { - printer.E("error: %v", err) - } else { - blobsLoaded = true - } - } - - printer.S(" ========================================") - printer.S(" inspect the pack itself") - - blobs, _, err := repo.ListPack(ctx, id, int64(len(buf))) - if err != nil { - return fmt.Errorf("pack %v: %v", id.Str(), err) - } - checkPackSize(blobs, len(buf), printer) - - if !blobsLoaded { - return loadBlobs(ctx, opts, repo, id, blobs, printer) - } - return nil -} - -func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { - // track current size and offset - var size, offset uint64 - - blobs.Sort() - - for _, pb := range blobs { - printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length) - if offset != uint64(pb.Offset) { - printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset) - } - offset = uint64(pb.Offset + pb.Length) - size += uint64(pb.Length) - } - size += uint64(pack.CalculateHeaderSize(blobs)) - - if uint64(fileSize) != size { - printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize) - } else { - printer.S(" file sizes match") - } -} diff --git a/internal/repository/debug.go b/internal/repository/debug.go new file mode 100644 index 000000000..7030ad718 --- /dev/null +++ b/internal/repository/debug.go @@ -0,0 +1,338 @@ +//go:build debug + +package repository + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "fmt" + "io" + "os" + "runtime" + "time" + + "github.com/klauspost/compress/zstd" + "golang.org/x/sync/errgroup" + + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" +) + +// DumpIndexes loads each on-disk index file and writes its debug dump to wr. +func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error { + return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error { + printer.S("index_id: %v", id) + if err != nil { + return err + } + + return idx.Dump(wr) + }) +} + +// ExaminePackOptions configures debug examination of a pack file. +type ExaminePackOptions struct { + TryRepair bool + RepairByte bool + ExtractPack bool + ReuploadBlobs bool +} + +// ExaminePack loads and inspects a pack file and its index entries. +func ExaminePack(ctx context.Context, repo restic.Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error { + printer.S("examine %v", id) + + buf, err := repo.LoadRaw(ctx, restic.PackFile, id) + // also process damaged pack files + if buf == nil { + return err + } + printer.S(" file size is %v", len(buf)) + gotID := restic.Hash(buf) + if !id.Equal(gotID) { + printer.S(" wanted hash %v, got %v", id, gotID) + } else { + printer.S(" hash for file content matches") + } + + printer.S(" ========================================") + printer.S(" looking for info in the indexes") + + blobsLoaded := false + // examine all data the indexes have for the pack file + for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { + blobs := b.Blobs + if len(blobs) == 0 { + continue + } + + checkPackSize(blobs, len(buf), printer) + + err = loadBlobs(ctx, opts, repo, id, blobs, printer) + if err != nil { + printer.E("error: %v", err) + } else { + blobsLoaded = true + } + } + + printer.S(" ========================================") + printer.S(" inspect the pack itself") + + blobs, _, err := repo.ListPack(ctx, id, int64(len(buf))) + if err != nil { + return fmt.Errorf("pack %v: %v", id.Str(), err) + } + checkPackSize(blobs, len(buf), printer) + + if !blobsLoaded { + return loadBlobs(ctx, opts, repo, id, blobs, printer) + } + return nil +} + +func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { + // track current size and offset + var size, offset uint64 + + blobs.Sort() + + for _, pb := range blobs { + printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length) + if offset != uint64(pb.Offset) { + printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset) + } + offset = uint64(pb.Offset + pb.Length) + size += uint64(pb.Length) + } + size += uint64(pack.CalculateHeaderSize(blobs)) + + if uint64(fileSize) != size { + printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize) + } else { + printer.S(" file sizes match") + } +} + +func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte { + if bytewise { + printer.S(" trying to repair blob by finding a broken byte") + } else { + printer.S(" trying to repair blob with single bit flip") + } + + ch := make(chan int) + var wg errgroup.Group + done := make(chan struct{}) + var fixed []byte + var found bool + + workers := runtime.GOMAXPROCS(0) + printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0)) + for i := 0; i < workers; i++ { + wg.Go(func() error { + // make a local copy of the buffer + buf := make([]byte, len(input)) + copy(buf, input) + + testFlip := func(idx int, pattern byte) bool { + // flip bits + buf[idx] ^= pattern + + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) + if err == nil { + printer.S("") + printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern) + printer.S(" hash is %v", restic.Hash(plaintext)) + close(done) + found = true + fixed = plaintext + return true + } + + // flip bits back + buf[idx] ^= pattern + return false + } + + for i := range ch { + if bytewise { + for j := 0; j < 255; j++ { + if testFlip(i, byte(j)) { + return nil + } + } + } else { + for j := 0; j < 7; j++ { + // flip each bit once + if testFlip(i, (1 << uint(j))) { + return nil + } + } + } + } + return nil + }) + } + + wg.Go(func() error { + defer close(ch) + + start := time.Now() + info := time.Now() + for i := range input { + select { + case ch <- i: + case <-done: + printer.S(" done after %v", time.Since(start)) + return nil + } + + if time.Since(info) > time.Second { + secs := time.Since(start).Seconds() + gps := float64(i) / secs + remaining := len(input) - i + eta := time.Duration(float64(remaining)/gps) * time.Second + + printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", + i, len(input), float32(i)/float32(len(input))*100, gps, eta) + info = time.Now() + } + } + return nil + }) + err := wg.Wait() + if err != nil { + panic("all go routines can only return nil") + } + + if !found { + printer.S("\n blob could not be repaired") + } + return fixed +} + +func decryptUnsigned(k *crypto.Key, buf []byte) []byte { + // strip signature at the end + l := len(buf) + nonce, ct := buf[:16], buf[16:l-16] + out := make([]byte, len(ct)) + + c, err := aes.NewCipher(k.EncryptionKey[:]) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + e := cipher.NewCTR(c, nonce) + e.XORKeyStream(out, ct) + + return out +} + +func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { + dec, err := zstd.NewReader(nil) + if err != nil { + panic(err) + } + + packData, err := repo.LoadRaw(ctx, restic.PackFile, packID) + // allow processing broken pack files + if packData == nil { + return err + } + + err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + for _, blob := range list { + printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length) + if int(blob.Offset+blob.Length) > len(packData) { + printer.E("skipping truncated blob") + continue + } + buf := packData[blob.Offset : blob.Offset+blob.Length] + key := repo.Key() + + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) + outputPrefix := "" + filePrefix := "" + if err != nil { + printer.E("error decrypting blob: %v", err) + if opts.TryRepair || opts.RepairByte { + plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer) + } + if plaintext != nil { + outputPrefix = "repaired " + filePrefix = "repaired-" + } else { + plaintext = decryptUnsigned(key, buf) + err = storePlainBlob(blob.ID, "damaged-", plaintext, printer) + if err != nil { + return err + } + continue + } + } + + if blob.IsCompressed() { + decompressed, err := dec.DecodeAll(plaintext, nil) + if err != nil { + printer.S(" failed to decompress blob %v", blob.ID) + } + if decompressed != nil { + plaintext = decompressed + } + } + + id := restic.Hash(plaintext) + var prefix string + if !id.Equal(blob.ID) { + printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID) + prefix = "wrong-hash-" + } else { + printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id) + prefix = "correct-" + } + if opts.ExtractPack { + err = storePlainBlob(id, filePrefix+prefix, plaintext, printer) + if err != nil { + return err + } + } + if opts.ReuploadBlobs { + _, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true) + if err != nil { + return err + } + printer.S(" uploaded %v %v", blob.Type, id) + } + } + return nil + }) + return err +} + +func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error { + filename := fmt.Sprintf("%s%s.bin", prefix, id) + f, err := os.Create(filename) + if err != nil { + return err + } + + _, err = f.Write(plain) + if err != nil { + _ = f.Close() + return err + } + + err = f.Close() + if err != nil { + return err + } + + printer.S("decrypt of blob %v stored at %v", id, filename) + return nil +}