diff --git a/.golangci.yml b/.golangci.yml index 5f52eb8c6..2365a5b04 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -40,6 +40,13 @@ linters: desc: "internal/restic should not be imported to keep the architectural layers intact" - pkg: "github.com/restic/restic/internal/repository" desc: "internal/repository should not be imported to keep the architectural layers intact" + repository-internals: + files: + - "**" + - "!**/internal/repository/**" + deny: + - pkg: "github.com/restic/restic/internal/repository/" + desc: "packages below internal/repository should not be imported to not depend on repository internals" importas: alias: - pkg: github.com/restic/restic/internal/test diff --git a/cmd/restic/cmd_copy_integration_test.go b/cmd/restic/cmd_copy_integration_test.go index 93f9685f6..252b2fdfe 100644 --- a/cmd/restic/cmd_copy_integration_test.go +++ b/cmd/restic/cmd_copy_integration_test.go @@ -103,7 +103,7 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i defer unlock() rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - blobs, _, err := repo.ListPack(context.TODO(), id, size) + blobs, err := repo.ListPack(context.TODO(), id, size) rtest.OK(t, err) rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob") diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 2d3a8d8d3..0a0b43295 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -4,28 +4,18 @@ package main import ( "context" - "crypto/aes" - "crypto/cipher" "encoding/json" "fmt" "io" - "os" - "runtime" "sync" - "time" - "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" "github.com/spf13/pflag" - "golang.org/x/sync/errgroup" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/global" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/repository/index" - "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/progress" @@ -147,7 +137,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, var m sync.Mutex return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { - blobs, _, err := repo.ListPack(ctx, id, size) + blobs, err := repo.ListPack(ctx, id, size) if err != nil { printer.E("error for pack %v: %v", id.Str(), err) return nil @@ -172,17 +162,6 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, }) } -func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error { - return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error { - printer.S("index_id: %v", id) - if err != nil { - return err - } - - return idx.Dump(wr) - }) -} - func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error { printer := ui.NewProgressPrinter(false, gopts.Verbosity, term) @@ -200,7 +179,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term switch tpe { case "indexes": - return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) + return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) case "snapshots": return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter()) case "packs": @@ -213,7 +192,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term } printer.S("indexes:") - err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) + err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) if err != nil { return err } @@ -224,225 +203,6 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term } } -func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte { - if bytewise { - printer.S(" trying to repair blob by finding a broken byte") - } else { - printer.S(" trying to repair blob with single bit flip") - } - - ch := make(chan int) - var wg errgroup.Group - done := make(chan struct{}) - var fixed []byte - var found bool - - workers := runtime.GOMAXPROCS(0) - printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0)) - for i := 0; i < workers; i++ { - wg.Go(func() error { - // make a local copy of the buffer - buf := make([]byte, len(input)) - copy(buf, input) - - testFlip := func(idx int, pattern byte) bool { - // flip bits - buf[idx] ^= pattern - - nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) - if err == nil { - printer.S("") - printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern) - printer.S(" hash is %v", restic.Hash(plaintext)) - close(done) - found = true - fixed = plaintext - return true - } - - // flip bits back - buf[idx] ^= pattern - return false - } - - for i := range ch { - if bytewise { - for j := 0; j < 255; j++ { - if testFlip(i, byte(j)) { - return nil - } - } - } else { - for j := 0; j < 7; j++ { - // flip each bit once - if testFlip(i, (1 << uint(j))) { - return nil - } - } - } - } - return nil - }) - } - - wg.Go(func() error { - defer close(ch) - - start := time.Now() - info := time.Now() - for i := range input { - select { - case ch <- i: - case <-done: - printer.S(" done after %v", time.Since(start)) - return nil - } - - if time.Since(info) > time.Second { - secs := time.Since(start).Seconds() - gps := float64(i) / secs - remaining := len(input) - i - eta := time.Duration(float64(remaining)/gps) * time.Second - - printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", - i, len(input), float32(i)/float32(len(input))*100, gps, eta) - info = time.Now() - } - } - return nil - }) - err := wg.Wait() - if err != nil { - panic("all go routines can only return nil") - } - - if !found { - printer.S("\n blob could not be repaired") - } - return fixed -} - -func decryptUnsigned(k *crypto.Key, buf []byte) []byte { - // strip signature at the end - l := len(buf) - nonce, ct := buf[:16], buf[16:l-16] - out := make([]byte, len(ct)) - - c, err := aes.NewCipher(k.EncryptionKey[:]) - if err != nil { - panic(fmt.Sprintf("unable to create cipher: %v", err)) - } - e := cipher.NewCTR(c, nonce) - e.XORKeyStream(out, ct) - - return out -} - -func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { - dec, err := zstd.NewReader(nil) - if err != nil { - panic(err) - } - - pack, err := repo.LoadRaw(ctx, restic.PackFile, packID) - // allow processing broken pack files - if pack == nil { - return err - } - - err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { - for _, blob := range list { - printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length) - if int(blob.Offset+blob.Length) > len(pack) { - printer.E("skipping truncated blob") - continue - } - buf := pack[blob.Offset : blob.Offset+blob.Length] - key := repo.Key() - - nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) - outputPrefix := "" - filePrefix := "" - if err != nil { - printer.E("error decrypting blob: %v", err) - if opts.TryRepair || opts.RepairByte { - plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer) - } - if plaintext != nil { - outputPrefix = "repaired " - filePrefix = "repaired-" - } else { - plaintext = decryptUnsigned(key, buf) - err = storePlainBlob(blob.ID, "damaged-", plaintext, printer) - if err != nil { - return err - } - continue - } - } - - if blob.IsCompressed() { - decompressed, err := dec.DecodeAll(plaintext, nil) - if err != nil { - printer.S(" failed to decompress blob %v", blob.ID) - } - if decompressed != nil { - plaintext = decompressed - } - } - - id := restic.Hash(plaintext) - var prefix string - if !id.Equal(blob.ID) { - printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID) - prefix = "wrong-hash-" - } else { - printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id) - prefix = "correct-" - } - if opts.ExtractPack { - err = storePlainBlob(id, filePrefix+prefix, plaintext, printer) - if err != nil { - return err - } - } - if opts.ReuploadBlobs { - _, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true) - if err != nil { - return err - } - printer.S(" uploaded %v %v", blob.Type, id) - } - } - return nil - }) - return err -} - -func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error { - filename := fmt.Sprintf("%s%s.bin", prefix, id) - f, err := os.Create(filename) - if err != nil { - return err - } - - _, err = f.Write(plain) - if err != nil { - _ = f.Close() - return err - } - - err = f.Close() - if err != nil { - return err - } - - printer.S("decrypt of blob %v stored at %v", id, filename) - return nil -} - func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error { printer := ui.NewProgressPrinter(false, gopts.Verbosity, term) @@ -478,8 +238,14 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin return err } + examineOpts := repository.ExaminePackOptions{ + TryRepair: opts.TryRepair, + RepairByte: opts.RepairByte, + ExtractPack: opts.ExtractPack, + ReuploadBlobs: opts.ReuploadBlobs, + } for _, id := range ids { - err := examinePack(ctx, opts, repo, id, printer) + err := repository.ExaminePack(ctx, repo, id, examineOpts, printer) if err != nil { printer.E("error: %v", err) } @@ -489,78 +255,3 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin } return nil } - -func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error { - printer.S("examine %v", id) - - buf, err := repo.LoadRaw(ctx, restic.PackFile, id) - // also process damaged pack files - if buf == nil { - return err - } - printer.S(" file size is %v", len(buf)) - gotID := restic.Hash(buf) - if !id.Equal(gotID) { - printer.S(" wanted hash %v, got %v", id, gotID) - } else { - printer.S(" hash for file content matches") - } - - printer.S(" ========================================") - printer.S(" looking for info in the indexes") - - blobsLoaded := false - // examine all data the indexes have for the pack file - for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { - blobs := b.Blobs - if len(blobs) == 0 { - continue - } - - checkPackSize(blobs, len(buf), printer) - - err = loadBlobs(ctx, opts, repo, id, blobs, printer) - if err != nil { - printer.E("error: %v", err) - } else { - blobsLoaded = true - } - } - - printer.S(" ========================================") - printer.S(" inspect the pack itself") - - blobs, _, err := repo.ListPack(ctx, id, int64(len(buf))) - if err != nil { - return fmt.Errorf("pack %v: %v", id.Str(), err) - } - checkPackSize(blobs, len(buf), printer) - - if !blobsLoaded { - return loadBlobs(ctx, opts, repo, id, blobs, printer) - } - return nil -} - -func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { - // track current size and offset - var size, offset uint64 - - blobs.Sort() - - for _, pb := range blobs { - printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length) - if offset != uint64(pb.Offset) { - printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset) - } - offset = uint64(pb.Offset + pb.Length) - size += uint64(pb.Length) - } - size += uint64(pack.CalculateHeaderSize(blobs)) - - if uint64(fileSize) != size { - printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize) - } else { - printer.S(" file sizes match") - } -} diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 6f7a6396a..5254b58a0 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -472,7 +472,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { delete(packIDs, idStr) } debug.Log("Found pack %s", idStr) - blobs, _, err := f.repo.ListPack(ctx, id, size) + blobs, err := f.repo.ListPack(ctx, id, size) if err != nil { return err } diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 1467962de..11482fab1 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -6,7 +6,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/global" - "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" @@ -69,18 +69,13 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T case "locks": t = restic.LockFile case "blobs": - return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error { - if err != nil { - return err + for entry := range repository.AllIndexBlobs(ctx, repo, repo) { + if entry.Error != nil { + return entry.Error } - for blobs := range idx.Values() { - if ctx.Err() != nil { - return ctx.Err() - } - printer.S("%v %v", blobs.Type, blobs.ID) - } - return nil - }) + printer.S("%v %v", entry.Handle.Type, entry.Handle.ID) + } + return nil default: return errors.Fatal("invalid type") } diff --git a/cmd/restic/cmd_repair_index_integration_test.go b/cmd/restic/cmd_repair_index_integration_test.go index 0a56f4b7a..a93e3df10 100644 --- a/cmd/restic/cmd_repair_index_integration_test.go +++ b/cmd/restic/cmd_repair_index_integration_test.go @@ -11,7 +11,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/global" - "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -61,15 +60,6 @@ func TestRebuildIndex(t *testing.T) { testRebuildIndex(t, nil) } -func TestRebuildIndexAlwaysFull(t *testing.T) { - indexFull := index.Full - defer func() { - index.Full = indexFull - }() - index.Full = func(*index.Index) bool { return true } - testRebuildIndex(t, nil) -} - // indexErrorBackend modifies the first index after reading. type indexErrorBackend struct { backend.Backend diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index eb53b14c2..4b2008bee 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -4,7 +4,6 @@ import ( "context" "io" "math/rand" - "os" "path/filepath" "sort" "strconv" @@ -19,7 +18,6 @@ import ( "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/repository/hashing" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -192,56 +190,19 @@ func TestModifiedIndex(t *testing.T) { Type: restic.IndexFile, Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } - - tmpfile, err := os.CreateTemp("", "restic-test-mod-index-") - if err != nil { - t.Fatal(err) - } - defer func() { - err := tmpfile.Close() - if err != nil { - t.Fatal(err) - } - - err = os.Remove(tmpfile.Name()) - if err != nil { - t.Fatal(err) - } - }() - wr := io.Writer(tmpfile) - var hw *hashing.Writer - if be.Hasher() != nil { - hw = hashing.NewWriter(wr, be.Hasher()) - wr = hw - } - - // read the file from the backend - err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error { - _, err := io.Copy(wr, rd) + var data []byte + test.OK(t, be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error { + var err error + data, err = io.ReadAll(rd) return err - }) - test.OK(t, err) - + })) // save the index again with a modified name so that the hash doesn't match // the content any more h2 := backend.Handle{ Type: restic.IndexFile, Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } - - var hash []byte - if hw != nil { - hash = hw.Sum(nil) - } - rd, err := backend.NewFileReader(tmpfile, hash) - if err != nil { - t.Fatal(err) - } - - err = be.Save(context.TODO(), h2, rd) - if err != nil { - t.Fatal(err) - } + test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher()))) chkr := checker.New(repo, false) hints, errs := chkr.LoadIndex(context.TODO(), nil) diff --git a/internal/repository/debug.go b/internal/repository/debug.go new file mode 100644 index 000000000..fd6533a82 --- /dev/null +++ b/internal/repository/debug.go @@ -0,0 +1,338 @@ +//go:build debug + +package repository + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "fmt" + "io" + "os" + "runtime" + "time" + + "github.com/klauspost/compress/zstd" + "golang.org/x/sync/errgroup" + + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" +) + +// DumpIndexes loads each on-disk index file and writes its debug dump to wr. +func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error { + return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error { + printer.S("index_id: %v", id) + if err != nil { + return err + } + + return idx.Dump(wr) + }) +} + +// ExaminePackOptions configures debug examination of a pack file. +type ExaminePackOptions struct { + TryRepair bool + RepairByte bool + ExtractPack bool + ReuploadBlobs bool +} + +// ExaminePack loads and inspects a pack file and its index entries. +func ExaminePack(ctx context.Context, repo restic.Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error { + printer.S("examine %v", id) + + buf, err := repo.LoadRaw(ctx, restic.PackFile, id) + // also process damaged pack files + if buf == nil { + return err + } + printer.S(" file size is %v", len(buf)) + gotID := restic.Hash(buf) + if !id.Equal(gotID) { + printer.S(" wanted hash %v, got %v", id, gotID) + } else { + printer.S(" hash for file content matches") + } + + printer.S(" ========================================") + printer.S(" looking for info in the indexes") + + blobsLoaded := false + // examine all data the indexes have for the pack file + for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { + blobs := b.Blobs + if len(blobs) == 0 { + continue + } + + checkPackSize(blobs, len(buf), printer) + + err = loadBlobs(ctx, opts, repo, id, blobs, printer) + if err != nil { + printer.E("error: %v", err) + } else { + blobsLoaded = true + } + } + + printer.S(" ========================================") + printer.S(" inspect the pack itself") + + blobs, err := repo.ListPack(ctx, id, int64(len(buf))) + if err != nil { + return fmt.Errorf("pack %v: %v", id.Str(), err) + } + checkPackSize(blobs, len(buf), printer) + + if !blobsLoaded { + return loadBlobs(ctx, opts, repo, id, blobs, printer) + } + return nil +} + +func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { + // track current size and offset + var size, offset uint64 + + blobs.Sort() + + for _, pb := range blobs { + printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length) + if offset != uint64(pb.Offset) { + printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset) + } + offset = uint64(pb.Offset + pb.Length) + size += uint64(pb.Length) + } + size += uint64(pack.CalculateHeaderSize(blobs)) + + if uint64(fileSize) != size { + printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize) + } else { + printer.S(" file sizes match") + } +} + +func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte { + if bytewise { + printer.S(" trying to repair blob by finding a broken byte") + } else { + printer.S(" trying to repair blob with single bit flip") + } + + ch := make(chan int) + var wg errgroup.Group + done := make(chan struct{}) + var fixed []byte + var found bool + + workers := runtime.GOMAXPROCS(0) + printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0)) + for i := 0; i < workers; i++ { + wg.Go(func() error { + // make a local copy of the buffer + buf := make([]byte, len(input)) + copy(buf, input) + + testFlip := func(idx int, pattern byte) bool { + // flip bits + buf[idx] ^= pattern + + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) + if err == nil { + printer.S("") + printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern) + printer.S(" hash is %v", restic.Hash(plaintext)) + close(done) + found = true + fixed = plaintext + return true + } + + // flip bits back + buf[idx] ^= pattern + return false + } + + for i := range ch { + if bytewise { + for j := 0; j < 255; j++ { + if testFlip(i, byte(j)) { + return nil + } + } + } else { + for j := 0; j < 7; j++ { + // flip each bit once + if testFlip(i, (1 << uint(j))) { + return nil + } + } + } + } + return nil + }) + } + + wg.Go(func() error { + defer close(ch) + + start := time.Now() + info := time.Now() + for i := range input { + select { + case ch <- i: + case <-done: + printer.S(" done after %v", time.Since(start)) + return nil + } + + if time.Since(info) > time.Second { + secs := time.Since(start).Seconds() + gps := float64(i) / secs + remaining := len(input) - i + eta := time.Duration(float64(remaining)/gps) * time.Second + + printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", + i, len(input), float32(i)/float32(len(input))*100, gps, eta) + info = time.Now() + } + } + return nil + }) + err := wg.Wait() + if err != nil { + panic("all go routines can only return nil") + } + + if !found { + printer.S("\n blob could not be repaired") + } + return fixed +} + +func decryptUnsigned(k *crypto.Key, buf []byte) []byte { + // strip signature at the end + l := len(buf) + nonce, ct := buf[:16], buf[16:l-16] + out := make([]byte, len(ct)) + + c, err := aes.NewCipher(k.EncryptionKey[:]) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + e := cipher.NewCTR(c, nonce) + e.XORKeyStream(out, ct) + + return out +} + +func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { + dec, err := zstd.NewReader(nil) + if err != nil { + panic(err) + } + + packData, err := repo.LoadRaw(ctx, restic.PackFile, packID) + // allow processing broken pack files + if packData == nil { + return err + } + + err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + for _, blob := range list { + printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length) + if int(blob.Offset+blob.Length) > len(packData) { + printer.E("skipping truncated blob") + continue + } + buf := packData[blob.Offset : blob.Offset+blob.Length] + key := repo.Key() + + nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] + plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil) + outputPrefix := "" + filePrefix := "" + if err != nil { + printer.E("error decrypting blob: %v", err) + if opts.TryRepair || opts.RepairByte { + plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer) + } + if plaintext != nil { + outputPrefix = "repaired " + filePrefix = "repaired-" + } else { + plaintext = decryptUnsigned(key, buf) + err = storePlainBlob(blob.ID, "damaged-", plaintext, printer) + if err != nil { + return err + } + continue + } + } + + if blob.IsCompressed() { + decompressed, err := dec.DecodeAll(plaintext, nil) + if err != nil { + printer.S(" failed to decompress blob %v", blob.ID) + } + if decompressed != nil { + plaintext = decompressed + } + } + + id := restic.Hash(plaintext) + var prefix string + if !id.Equal(blob.ID) { + printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID) + prefix = "wrong-hash-" + } else { + printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id) + prefix = "correct-" + } + if opts.ExtractPack { + err = storePlainBlob(id, filePrefix+prefix, plaintext, printer) + if err != nil { + return err + } + } + if opts.ReuploadBlobs { + _, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true) + if err != nil { + return err + } + printer.S(" uploaded %v %v", blob.Type, id) + } + } + return nil + }) + return err +} + +func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error { + filename := fmt.Sprintf("%s%s.bin", prefix, id) + f, err := os.Create(filename) + if err != nil { + return err + } + + _, err = f.Write(plain) + if err != nil { + _ = f.Close() + return err + } + + err = f.Close() + if err != nil { + return err + } + + printer.S("decrypt of blob %v stored at %v", id, filename) + return nil +} diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index fb5344659..56d01a503 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -694,3 +694,62 @@ func TestRewriteSplitPacks(t *testing.T) { blobs := mi.Lookup(blobOther.BlobHandle) rtest.Equals(t, nil, blobs) } + +// TestRewriteFullPacks checks that Rewrite drops a duplicate full index for the same +// pack while keeping the other index files and blob lookups intact. Creates 3 indexes: +// - indexA: contains packA +// - indexB: contains packB +// - indexC: contains packB +// After the rewrite, indexC must be dropped. The other indexes must be kept. +func TestRewriteFullPacks(t *testing.T) { + originalFull := index.Full + defer func() { + index.Full = originalFull + }() + index.Full = func(*index.Index) bool { return true } + + repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion) + + packA := restic.NewRandomID() + packB := restic.NewRandomID() + + blobA := restic.PackedBlob{ + PackID: packA, + Blob: restic.Blob{ + BlobHandle: restic.NewRandomBlobHandle(), + Length: uint(crypto.CiphertextLength(10)), + Offset: 0, + }, + } + blobB := restic.PackedBlob{ + PackID: packB, + Blob: restic.Blob{ + BlobHandle: restic.NewRandomBlobHandle(), + Length: uint(crypto.CiphertextLength(50)), + Offset: 0, + }, + } + + mi := index.NewMasterIndex() + rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked)) + rtest.OK(t, mi.Flush(context.TODO(), unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.Flush(context.TODO(), unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.Flush(context.TODO(), unpacked)) + + indexIDs := mi.IDs() + rtest.Equals(t, 3, len(indexIDs)) + + rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{})) + + mi2 := index.NewMasterIndex() + rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil)) + + afterRewrite := mi2.IDs() + rtest.Equals(t, 2, len(afterRewrite)) + rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs))) + + rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle)) + rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle)) +} diff --git a/internal/repository/index_list.go b/internal/repository/index_list.go new file mode 100644 index 000000000..18b40f114 --- /dev/null +++ b/internal/repository/index_list.go @@ -0,0 +1,41 @@ +package repository + +import ( + "context" + "iter" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/restic" +) + +// IndexBlob is one blob handle from an on-disk index file, or an error from loading/decoding +// that file. +type IndexBlob struct { + Handle restic.BlobHandle + Error error +} + +// AllIndexBlobs streams blob handles from each index file without building a master index. +func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.LoaderUnpacked) iter.Seq[IndexBlob] { + return func(yield func(IndexBlob) bool) { + stopIteration := errors.New("stop index blob iteration") + err := index.ForAllIndexes(ctx, lister, loader, func(_ restic.ID, idx *index.Index, err error) error { + if err != nil { + return err + } + for blob := range idx.Values() { + if ctx.Err() != nil { + return ctx.Err() + } + if !yield(IndexBlob{Handle: blob.BlobHandle}) { + return stopIteration + } + } + return nil + }) + if err != nil && !errors.Is(err, stopIteration) { + yield(IndexBlob{Error: err}) + } + } +} diff --git a/internal/repository/index_list_test.go b/internal/repository/index_list_test.go new file mode 100644 index 000000000..2a9052879 --- /dev/null +++ b/internal/repository/index_list_test.go @@ -0,0 +1,62 @@ +package repository_test + +import ( + "context" + "testing" + + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func TestAllIndexBlobs(t *testing.T) { + repo, _, _ := repository.TestRepositoryWithVersion(t, 0) + + want := restic.NewBlobSet() + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + for i := range 5 { + data := []byte{byte('a' + i)} + id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, data, restic.ID{}, false) + rtest.OK(t, err) + want.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id}) + } + return nil + })) + + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) + + fromMaster := restic.NewBlobSet() + rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { + fromMaster.Insert(pb.BlobHandle) + })) + rtest.Equals(t, want, fromMaster) + + fromStream := restic.NewBlobSet() + for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) { + if entry.Error != nil { + t.Fatalf("unexpected error: %v", entry.Error) + } + fromStream.Insert(entry.Handle) + } + rtest.Equals(t, want, fromStream) +} + +func TestAllIndexBlobsEarlyStop(t *testing.T) { + repo, _, _ := repository.TestRepositoryWithVersion(t, 0) + + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + for range 5 { + _, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, []byte("test"), restic.ID{}, false) + rtest.OK(t, err) + } + return nil + })) + + var count int + for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) { + rtest.Assert(t, entry.Error == nil, "unexpected error after early stop: %v", entry.Error) + count++ + break + } + rtest.Equals(t, 1, count) +} diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 0c1095301..fc7d86900 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -86,7 +86,7 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa blobs := restic.NewBlobSet() err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - entries, _, err := repo.ListPack(context.TODO(), id, size) + entries, err := repo.ListPack(context.TODO(), id, size) if err != nil { t.Fatalf("error listing pack %v: %v", id, err) } diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index 39d057976..5059aad04 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -76,7 +76,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { if ids.Has(id) { - blobs, _, err := repo.ListPack(ctx, id, size) + blobs, err := repo.ListPack(ctx, id, size) if err != nil { return nil } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 86eab8183..7bc61bcc0 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -781,7 +781,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest // a worker receives an pack ID from ch, reads the pack contents, and adds them to idx worker := func() error { for fi := range ch { - entries, _, err := r.ListPack(wgCtx, fi.ID, fi.Size) + entries, err := r.ListPack(wgCtx, fi.ID, fi.Size) if err != nil { debug.Log("unable to list pack file %v", fi.ID.Str()) m.Lock() @@ -957,12 +957,11 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic }) } -// ListPack returns the list of blobs saved in the pack id and the length of -// the pack header. -func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, uint32, error) { +// ListPack returns the list of blobs saved in the pack id. +func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) { h := backend.Handle{Type: restic.PackFile, Name: id.String()} - entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) + entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) if err != nil { if r.cache != nil { // ignore error as there is not much we can do here @@ -970,9 +969,9 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re } // retry on error - entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) + entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) } - return entries, hdrSize, err + return entries, err } // Delete calls backend.Delete() if implemented, and returns an error diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 7bb89d385..dc4febaaf 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -479,7 +479,7 @@ func TestListPack(t *testing.T) { return nil })) - blobs, _, err := repo.ListPack(context.TODO(), packID, size) + blobs, err := repo.ListPack(context.TODO(), packID, size) rtest.OK(t, err) rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs) diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 1476c2468..f8b5cef33 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -32,9 +32,8 @@ type Repository interface { // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. ListBlobs(ctx context.Context, fn func(PackedBlob)) error ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs - // ListPack returns the list of blobs saved in the pack id and the length of - // the pack header. - ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, hdrSize uint32, err error) + // ListPack returns the list of blobs saved in the pack id. + ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error) LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error) LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error