From f186e1e4589496e674f80d200a5745be32c5eb83 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 Jun 2026 22:21:38 +0200 Subject: [PATCH 1/8] repository: require *Repository for CopyBlobs Prepare `CopyBlobs` to allow access to unexported methods of the Repository struct. This requires changing the test to inject the number of backend connections via a wrapped backend instead of a wrapped repository. --- cmd/restic/cmd_copy.go | 4 ++-- internal/repository/repack.go | 4 ++-- internal/repository/repack_test.go | 31 ++++++++++++++++++------------ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 7728af1a2..28c0a2dfb 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -189,7 +189,7 @@ func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool { // copyTreeBatched copies multiple snapshots in one go. Snapshots are written after // data equivalent to at least 10 packfiles was written. -func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, +func copyTreeBatched(ctx context.Context, srcRepo *repository.Repository, dstRepo restic.Repository, selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error { // remember already processed trees across all snapshots @@ -254,7 +254,7 @@ func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo res return nil } -func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, +func copyTree(ctx context.Context, srcRepo *repository.Repository, dstRepo restic.Repository, visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) { copyBlobs := srcRepo.NewAssociatedBlobSet() diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 2793348b8..431aefd69 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -30,7 +30,7 @@ type LogFunc func(msg string, args ...interface{}) // blobs have been processed. func CopyBlobs( ctx context.Context, - repo restic.Repository, + repo *Repository, dstRepo restic.Repository, dstUploader restic.BlobSaverWithAsync, packs restic.IDSet, @@ -55,7 +55,7 @@ func CopyBlobs( func repack( ctx context.Context, - repo restic.Repository, + repo *Repository, dstRepo restic.Repository, uploader restic.BlobSaverWithAsync, packs restic.IDSet, diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index c490759ba..0488334db 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -149,7 +149,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe return packs } -func repack(t *testing.T, repo restic.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) { +func repack(t *testing.T, repo *repository.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) { rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, nil, nil) })) @@ -238,21 +238,28 @@ func TestRepackCopy(t *testing.T) { repository.TestAllVersions(t, testRepackCopy) } -type oneConnectionRepo struct { - restic.Repository +// oneConnectionBackend limits concurrent backend operations to test repack with +// the minimum connection count required by CopyBlobs. +type oneConnectionBackend struct { + backend.Backend } -func (r oneConnectionRepo) Connections() uint { - return 1 +func (be *oneConnectionBackend) Properties() backend.Properties { + p := be.Backend.Properties() + p.Connections = 1 + return p +} + +func (be *oneConnectionBackend) Unwrap() backend.Backend { + return be.Backend } func testRepackCopy(t *testing.T, version uint) { - repo, _, _ := repository.TestRepositoryWithVersion(t, version) - dstRepo, _, _ := repository.TestRepositoryWithVersion(t, version) - // test with minimal possible connection count - repoWrapped := &oneConnectionRepo{repo} - dstRepoWrapped := &oneConnectionRepo{dstRepo} + repo, _ := repository.TestRepositoryWithBackend(t, &oneConnectionBackend{Backend: repository.TestBackend(t)}, version, repository.Options{}) + dstRepo, _ := repository.TestRepositoryWithBackend(t, &oneConnectionBackend{Backend: repository.TestBackend(t)}, version, repository.Options{}) + rtest.Equals(t, repo.Connections(), 1) + rtest.Equals(t, dstRepo.Connections(), 1) seed := time.Now().UnixNano() random := rand.New(rand.NewSource(seed)) @@ -265,8 +272,8 @@ func testRepackCopy(t *testing.T, version uint) { _, keepBlobs := selectBlobs(t, random, repo, 0.2) copyPacks := findPacksForBlobs(t, repo, keepBlobs) - rtest.OK(t, repoWrapped.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { - return repository.CopyBlobs(ctx, repoWrapped, dstRepoWrapped, uploader, copyPacks, keepBlobs, nil, nil) + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return repository.CopyBlobs(ctx, repo, dstRepo, uploader, copyPacks, keepBlobs, nil, nil) })) rebuildAndReloadIndex(t, dstRepo) From 8169814b38ec9869f96078412b1e19d76d1edf7c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 5 Jun 2026 00:01:24 +0200 Subject: [PATCH 2/8] restic: switch LoadBlobsFromPack to BlobHandles LoadBlobsFromPack now resolves the handles to Blobs. Repository internal code can still use the Blob-based method. The loader used in the filerestorer test now has to implement sorting the blobs by offset itself as it no longer has access to the repository-internal dataypes. --- internal/repository/repack.go | 2 +- internal/repository/repair_pack.go | 2 +- internal/repository/repository.go | 28 ++++++++++++++++++- internal/restic/repository.go | 2 +- internal/restorer/filerestorer.go | 14 +++++----- internal/restorer/filerestorer_test.go | 38 ++++++++++++++++---------- 6 files changed, 60 insertions(+), 26 deletions(-) diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 431aefd69..6900eb96c 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -106,7 +106,7 @@ func repack( worker := func() error { for t := range downloadQueue { - err := repo.LoadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + err := repo.loadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { // a required blob couldn't be retrieved return err diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index 5059aad04..735653a4c 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -91,7 +91,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe } func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs restic.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error { - err := repo.LoadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + err := repo.loadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { printer.E("failed to load blob %v: %v", blob.ID, err) return nil diff --git a/internal/repository/repository.go b/internal/repository/repository.go index eb6c0958b..360de6b50 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -1054,7 +1054,33 @@ const maxUnusedRange = 1 * 1024 * 1024 // handleBlobFn is called at most once for each blob. If the callback returns an error, // then LoadBlobsFromPack will abort and not retry it. The buf passed to the callback is only valid within // this specific call. The callback must not keep a reference to buf. -func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + blobs, err := r.blobsInPack(packID, handles) + if err != nil { + return err + } + return r.loadBlobsFromPack(ctx, packID, blobs, handleBlobFn) +} + +func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (restic.Blobs, error) { + blobs := make(restic.Blobs, 0, len(handles)) + for _, h := range handles { + found := false + for _, pb := range r.idx.Lookup(h) { + if pb.PackID.Equal(packID) { + blobs = append(blobs, pb.Blob) + found = true + break + } + } + if !found { + return nil, errors.Errorf("blob %v not found in pack %v", h, packID) + } + } + return blobs, nil +} + +func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return streamPack(ctx, r.be.Load, r.LoadBlob, r.getZstdDecoder(), r.key, packID, blobs, handleBlobFn) } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index f8b5cef33..09693621d 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -36,7 +36,7 @@ type Repository interface { ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error) LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error) - LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error + LoadBlobsFromPack(ctx context.Context, packID ID, blobs []BlobHandle, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error // WithUploader starts the necessary workers to upload new blobs. Once the callback returns, // the workers are stopped and the index is written to the repository. The callback must use diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 63dc3d6e9..5207f4f8d 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -42,7 +42,7 @@ type packInfo struct { files map[*fileInfo]struct{} // set of files that use blobs from this pack } -type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error +type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error type startWarmupFn func(context.Context, restic.IDSet) (restic.WarmupJob, error) // fileRestorer restores set of files @@ -261,14 +261,14 @@ func (r *fileRestorer) truncateFileToSize(location string, size int64) error { type blobToFileOffsetsMapping map[restic.ID]struct { files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file - blob restic.Blob + blob restic.BlobHandle } func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // calculate blob->[]files->[]offsets mappings blobs := make(blobToFileOffsetsMapping) for file := range pack.files { - addBlob := func(blob restic.Blob, fileOffset int64) { + addBlob := func(blob restic.BlobHandle, fileOffset int64) { blobInfo, ok := blobs[blob.ID] if !ok { blobInfo.files = make(map[*fileInfo][]int64) @@ -280,7 +280,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { if fileBlobs, ok := file.blobs.(restic.IDs); ok { err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) { - addBlob(blob, fileOffset) + addBlob(blob.BlobHandle, fileOffset) } }) if err != nil { @@ -292,7 +292,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { idxPacks := r.idx(restic.DataBlob, blob.id) for _, idxPack := range idxPacks { if idxPack.PackID.Equal(pack.id) { - addBlob(idxPack.Blob, blob.offset) + addBlob(idxPack.BlobHandle, blob.offset) break } } @@ -324,7 +324,7 @@ func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlob // only report error for not yet processed blobs affectedFiles := make(map[*fileInfo]struct{}) for _, entry := range blobs { - if processedBlobs.Has(entry.blob.BlobHandle) { + if processedBlobs.Has(entry.blob) { continue } for file := range entry.files { @@ -343,7 +343,7 @@ func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlob func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { - blobList := make(restic.Blobs, 0, len(blobs)) + blobList := make([]restic.BlobHandle, 0, len(blobs)) for _, entry := range blobs { blobList = append(blobList, entry.blob) } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index 120288378..f5220998d 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -2,9 +2,11 @@ package restorer import ( "bytes" + "cmp" "context" "fmt" "os" + "slices" "testing" "github.com/restic/restic/internal/errors" @@ -135,24 +137,30 @@ func newTestRepo(content []TestFile) *TestRepo { filesPathToContent: filesPathToContent, warmupJobs: []*TestWarmupJob{}, } - repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { - blobs = append(restic.Blobs{}, blobs...) - blobs.Sort() - - for _, blob := range blobs { + repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + entries := make([]restic.PackedBlob, 0, len(handles)) + for _, h := range handles { found := false - for _, e := range repo.blobs[blob.ID] { + for _, e := range repo.blobs[h.ID] { if packID == e.PackID { + entries = append(entries, e) found = true - buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] - err := handleBlobFn(e.BlobHandle, buf, nil) - if err != nil { - return err - } + break } } if !found { - return fmt.Errorf("missing blob: %v", blob) + return fmt.Errorf("missing blob: %v", h) + } + } + slices.SortFunc(entries, func(a, b restic.PackedBlob) int { + return cmp.Compare(a.Offset, b.Offset) + }) + + for _, e := range entries { + buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] + err := handleBlobFn(e.BlobHandle, buf, nil) + if err != nil { + return err } } return nil @@ -313,7 +321,7 @@ func TestErrorRestoreFiles(t *testing.T) { loadError := errors.New("load error") // loader always returns an error - repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return loadError } @@ -346,9 +354,9 @@ func TestFatalDownloadError(t *testing.T) { repo := newTestRepo(content) loader := repo.loader - repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { ctr := 0 - return loader(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + return loader(ctx, packID, handles, func(blob restic.BlobHandle, buf []byte, err error) error { if ctr < 2 { ctr++ return handleBlobFn(blob, buf, err) From a9e0b463580eac9bf934aaa2e51b6d5399698fc2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 Jun 2026 21:58:23 +0200 Subject: [PATCH 3/8] restic: list pack header via ListPackHandles Replace ListPack with ListPackHandles so callers only receive blob handles from pack headers, not layout fields. --- cmd/restic/cmd_copy_integration_test.go | 8 ++++---- cmd/restic/cmd_find.go | 12 ++++++------ internal/repository/debug.go | 4 ++-- internal/repository/repack_test.go | 9 ++++----- internal/repository/repair_pack.go | 2 +- internal/repository/repository.go | 19 ++++++++++++++++--- internal/repository/repository_test.go | 6 +++--- internal/restic/repository.go | 4 ++-- 8 files changed, 38 insertions(+), 26 deletions(-) diff --git a/cmd/restic/cmd_copy_integration_test.go b/cmd/restic/cmd_copy_integration_test.go index 29e269578..55c7dbb7e 100644 --- a/cmd/restic/cmd_copy_integration_test.go +++ b/cmd/restic/cmd_copy_integration_test.go @@ -103,17 +103,17 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i defer unlock() rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - blobs, err := repo.ListPack(context.TODO(), id, size) + handles, err := repo.ListPackHandles(context.TODO(), id, size) rtest.OK(t, err) - rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob") + rtest.Assert(t, len(handles) > 0, "a packfile should contain at least one blob") - switch blobs[0].Type { + switch handles[0].Type { case restic.TreeBlob: countTreePacks++ case restic.DataBlob: countDataPacks++ } - countBlobs += len(blobs) + countBlobs += len(handles) return nil })) return nil diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index baa47dc1f..1502e897b 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -473,18 +473,18 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { delete(packIDs, idStr) } debug.Log("Found pack %s", idStr) - blobs, err := f.repo.ListPack(ctx, id, size) + handles, err := f.repo.ListPackHandles(ctx, id, size) if err != nil { return err } - for _, b := range blobs { - switch b.Type { + for _, h := range handles { + switch h.Type { case restic.DataBlob: - f.blobIDs[b.ID.String()] = struct{}{} + f.blobIDs[h.ID.String()] = struct{}{} case restic.TreeBlob: - f.treeIDs[b.ID.String()] = struct{}{} + f.treeIDs[h.ID.String()] = struct{}{} default: - panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String())) + panic(fmt.Sprintf("unknown type %v in blob list", h.Type.String())) } } // Stop searching when all packs have been found diff --git a/internal/repository/debug.go b/internal/repository/debug.go index c6691380c..29cb5b672 100644 --- a/internal/repository/debug.go +++ b/internal/repository/debug.go @@ -49,7 +49,7 @@ func writePackDumpJSON(wr io.Writer, item any) error { func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error { var m sync.Mutex return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { - blobs, err := repo.ListPack(ctx, id, size) + blobs, err := repo.listPack(ctx, id, size) if err != nil { printer.E("error for pack %v: %v", id.Str(), err) return nil @@ -135,7 +135,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami printer.S(" ========================================") printer.S(" inspect the pack itself") - blobs, err := repo.ListPack(ctx, id, int64(len(buf))) + blobs, err := repo.listPack(ctx, id, int64(len(buf))) if err != nil { return fmt.Errorf("pack %v: %v", id.Str(), err) } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 0488334db..4ead80153 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -86,13 +86,12 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa blobs := restic.NewBlobSet() err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - entries, err := repo.ListPack(context.TODO(), id, size) + handles, err := repo.ListPackHandles(context.TODO(), id, size) if err != nil { t.Fatalf("error listing pack %v: %v", id, err) } - for _, entry := range entries { - h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} + for _, h := range handles { if blobs.Has(h) { t.Errorf("ignoring duplicate blob %v", h) return nil @@ -100,9 +99,9 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa blobs.Insert(h) if random.Float32() <= p { - list1.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) + list1.Insert(h) } else { - list2.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) + list2.Insert(h) } } return nil diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index 735653a4c..bff83b846 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -76,7 +76,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { if ids.Has(id) { - blobs, err := repo.ListPack(ctx, id, size) + blobs, err := repo.listPack(ctx, id, size) if err != nil { return nil } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 360de6b50..ea519afa5 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -783,7 +783,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest // a worker receives an pack ID from ch, reads the pack contents, and adds them to idx worker := func() error { for fi := range ch { - entries, err := r.ListPack(wgCtx, fi.ID, fi.Size) + entries, err := r.listPack(wgCtx, fi.ID, fi.Size) if err != nil { debug.Log("unable to list pack file %v", fi.ID.Str()) m.Lock() @@ -964,8 +964,8 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic }) } -// ListPack returns the list of blobs saved in the pack id. -func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) { +// listPack returns blob entries from the pack file header including offsets. +func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) { h := backend.Handle{Type: restic.PackFile, Name: id.String()} entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) @@ -981,6 +981,19 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re return entries, err } +// ListPackHandles returns the blob handles stored in the pack file header. +func (r *Repository) ListPackHandles(ctx context.Context, id restic.ID, size int64) ([]restic.BlobHandle, error) { + blobs, err := r.listPack(ctx, id, size) + if err != nil { + return nil, err + } + handles := make([]restic.BlobHandle, len(blobs)) + for i, blob := range blobs { + handles[i] = blob.BlobHandle + } + return handles, nil +} + // Delete calls backend.Delete() if implemented, and returns an error // otherwise. func (r *Repository) Delete(ctx context.Context) error { diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index dc4febaaf..a35bd36d7 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -479,11 +479,11 @@ func TestListPack(t *testing.T) { return nil })) - blobs, err := repo.ListPack(context.TODO(), packID, size) + handles, err := repo.ListPackHandles(context.TODO(), packID, size) rtest.OK(t, err) - rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs) + rtest.Assert(t, len(handles) == 1 && handles[0].ID == id, "unexpected blobs in pack: %v", handles) - rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as ListPack does not set IsMetadata in the backend.Handle") + rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as listPack does not set IsMetadata in the backend.Handle") } func TestNoDoubleInit(t *testing.T) { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 09693621d..561755c6c 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -32,8 +32,8 @@ type Repository interface { // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. ListBlobs(ctx context.Context, fn func(PackedBlob)) error ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs - // ListPack returns the list of blobs saved in the pack id. - ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error) + // ListPackHandles returns the blob handles stored in the pack file header. + ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error) LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error) LoadBlobsFromPack(ctx context.Context, packID ID, blobs []BlobHandle, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error From c060c317d3cebbbe8f0e316fc182b6e1ee826482 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 Jun 2026 21:59:12 +0200 Subject: [PATCH 4/8] repository: unexport listPacksFromIndex `ListPacksFromIndex` only has repository-internal callers left (besides test code). --- internal/repository/checker.go | 2 +- internal/repository/checker_test.go | 2 +- internal/repository/debug.go | 9 ++++----- internal/repository/repack.go | 2 +- internal/repository/repair_pack.go | 2 +- internal/repository/repair_pack_test.go | 18 ++++++++---------- internal/repository/repository.go | 3 ++- internal/restic/repository.go | 1 - 8 files changed, 18 insertions(+), 21 deletions(-) diff --git a/internal/repository/checker.go b/internal/repository/checker.go index c3f530abf..845048b4a 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -306,7 +306,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID } // push packs to ch - for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) { + for pbs := range c.repo.listPacksFromIndex(ctx, packSet) { size := packs[pbs.PackID] debug.Log("listed %v", pbs.PackID) select { diff --git a/internal/repository/checker_test.go b/internal/repository/checker_test.go index def9a9951..e11584192 100644 --- a/internal/repository/checker_test.go +++ b/internal/repository/checker_test.go @@ -47,7 +47,7 @@ func TestGapInBlobs(t *testing.T) { rtest.Assert(t, ok, "expected pack 19a731a515618ec8b75fc0ff3b887d8feb83aef1001c9899f6702761142ed068") blobs := []restic.Blob{} - pb := <-repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(packID)) + pb := <-repo.listPacksFromIndex(context.TODO(), restic.NewIDSet(packID)) blobs = append(blobs, pb.Blobs...) // assertion for clarity, actually can't fail as the packfile content is fixed diff --git a/internal/repository/debug.go b/internal/repository/debug.go index 29cb5b672..8bf25a715 100644 --- a/internal/repository/debug.go +++ b/internal/repository/debug.go @@ -116,15 +116,14 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami blobsLoaded := false // examine all data the indexes have for the pack file - for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { - blobs := b.Blobs - if len(blobs) == 0 { + for b := range repo.listPacksFromIndex(ctx, restic.NewIDSet(id)) { + if len(b.Blobs) == 0 { continue } - checkPackSize(blobs, len(buf), printer) + checkPackSize(b.Blobs, len(buf), printer) - err = loadBlobs(ctx, opts, repo, id, blobs, printer) + err = loadBlobs(ctx, opts, repo, id, b.Blobs, printer) if err != nil { printer.E("error: %v", err) } else { diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 6900eb96c..a4472c91e 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -83,7 +83,7 @@ func repack( downloadQueue := make(chan restic.PackBlobs) wg.Go(func() error { defer close(downloadQueue) - for pbs := range repo.ListPacksFromIndex(wgCtx, packs) { + for pbs := range repo.listPacksFromIndex(wgCtx, packs) { var packBlobs restic.Blobs keepMutex.Lock() // filter out unnecessary blobs diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index bff83b846..a7d04bbc2 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -23,7 +23,7 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // examine all data the indexes have for the pack file - for b := range repo.ListPacksFromIndex(ctx, ids) { + for b := range repo.listPacksFromIndex(ctx, ids) { indexBlobs := b.Blobs err := reuploadBlobsFromPack(ctx, repo, b.PackID, indexBlobs, printer, uploader) if err != nil { diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index 558889e0d..c44839860 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -66,13 +66,11 @@ func testRepairBrokenPack(t *testing.T, version uint) { // find blob that starts at offset 0 var damagedBlob restic.BlobHandle - for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { - for _, blob := range blobs.Blobs { - if blob.Offset == 0 { - damagedBlob = blob.BlobHandle - } + _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { + if pb.PackID == damagedID && pb.Offset == 0 { + damagedBlob = pb.BlobHandle } - } + }) return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob) }, @@ -89,11 +87,11 @@ func testRepairBrokenPack(t *testing.T, version uint) { // all blobs in the file are broken damagedBlobs := restic.NewBlobSet() - for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { - for _, blob := range blobs.Blobs { - damagedBlobs.Insert(blob.BlobHandle) + _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { + if pb.PackID == damagedID { + damagedBlobs.Insert(pb.BlobHandle) } - } + }) return restic.NewIDSet(damagedID), damagedBlobs }, }, { diff --git a/internal/repository/repository.go b/internal/repository/repository.go index ea519afa5..36fa0e22e 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -693,7 +693,8 @@ func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) return nil } -func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { +// listPacksFromIndex returns index entries for the given packs, grouped by pack file. +func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { return r.idx.ListPacks(ctx, packs) } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 561755c6c..7986e2f74 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -31,7 +31,6 @@ type Repository interface { // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. ListBlobs(ctx context.Context, fn func(PackedBlob)) error - ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs // ListPackHandles returns the blob handles stored in the pack file header. ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error) From 97f1e99ed9be1640d28d013a6c6977dd789f71eb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 Jun 2026 21:59:48 +0200 Subject: [PATCH 5/8] restic: add PackBlob interface and implement it on PackedBlob The PackBlob interface will allow hiding details from the public interface, in particular, the offset of a blob within a pack file. --- internal/restic/blob.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/internal/restic/blob.go b/internal/restic/blob.go index ba9277aac..8d1e8a720 100644 --- a/internal/restic/blob.go +++ b/internal/restic/blob.go @@ -41,12 +41,42 @@ func (b Blobs) Sort() { }) } +// PackBlob is one index entry for a blob in a pack file. +// The interface intentionally omits the offset at which a blob is stored in the pack. +// This ensures that pack file internals are not leaked. +type PackBlob interface { + PackID() ID + Handle() BlobHandle + // CiphertextLength is the encrypted size stored in the pack. + CiphertextLength() uint + // PlaintextLength is the size after decryption/decompression. + PlaintextLength() uint + IsCompressed() bool +} + // PackedBlob is a blob stored within a file. type PackedBlob struct { Blob PackID ID } +type packBlob struct { + PackedBlob +} + +func (pb packBlob) PackID() ID { return pb.PackedBlob.PackID } + +func (pb packBlob) Handle() BlobHandle { return pb.BlobHandle } + +func (pb packBlob) CiphertextLength() uint { return pb.Length } + +func (pb packBlob) PlaintextLength() uint { return pb.DataLength() } + +func (pb packBlob) IsCompressed() bool { return pb.Blob.IsCompressed() } + +// AsPackBlob returns a PackBlob view of a PackedBlob. +func AsPackBlob(pb PackedBlob) PackBlob { return packBlob{pb} } + // BlobHandle identifies a blob of a given type. type BlobHandle struct { ID ID From ccb5ae15923eee0c4b26d4fe8ccd7d7f30bf8f2e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 5 Jun 2026 11:21:02 +0200 Subject: [PATCH 6/8] restic: change ListBlobs to return PackBlob PackBlob is a limited interface that only exposes a part of the information provided by PackedBlob. Most of the changes are switches from direct value lookups to the interface methods, with a few larger changes to let the tests still work. --- cmd/restic/cmd_find.go | 11 +++--- cmd/restic/cmd_find_integration_test.go | 15 +++++---- cmd/restic/cmd_list_integration_test.go | 4 +-- cmd/restic/cmd_recover.go | 7 ++-- cmd/restic/cmd_stats.go | 4 +-- cmd/restic/integration_helpers_test.go | 12 +++---- internal/checker/checker.go | 4 +-- internal/repository/checker.go | 12 ++++--- internal/repository/index_list_test.go | 4 +-- internal/repository/index_testutil_test.go | 17 ++++++++++ internal/repository/pack/pack.go | 17 +++++----- internal/repository/prune.go | 39 ++++++++++++---------- internal/repository/repair_pack_test.go | 21 ++++++------ internal/repository/repository.go | 4 +-- internal/restic/repository.go | 4 +-- 15 files changed, 102 insertions(+), 73 deletions(-) create mode 100644 internal/repository/index_testutil_test.go diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 1502e897b..df6363d21 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -526,22 +526,23 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc // remember which packs were found in the index indexPackIDs := make(map[string]struct{}) - err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) { - idStr := pb.PackID.String() + err := f.repo.ListBlobs(wctx, func(pb restic.PackBlob) { + packID := pb.PackID() + idStr := packID.String() // keep entry in packIDs as Each() returns individual index entries matchingID := false if _, ok := packIDs[idStr]; ok { matchingID = true } else { - if _, ok := packIDs[pb.PackID.Str()]; ok { + if _, ok := packIDs[packID.Str()]; ok { // expand id - delete(packIDs, pb.PackID.Str()) + delete(packIDs, packID.Str()) packIDs[idStr] = struct{}{} matchingID = true } } if matchingID { - f.blobIDs[pb.ID.String()] = struct{}{} + f.blobIDs[pb.Handle().ID.String()] = struct{}{} indexPackIDs[idStr] = struct{}{} } }) diff --git a/cmd/restic/cmd_find_integration_test.go b/cmd/restic/cmd_find_integration_test.go index e5e35348f..2587c2e01 100644 --- a/cmd/restic/cmd_find_integration_test.go +++ b/cmd/restic/cmd_find_integration_test.go @@ -179,9 +179,10 @@ func TestFindPackfile(t *testing.T) { packID := restic.ID{} done := false - err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) { - if !done && pb.Type == restic.TreeBlob { - packID = pb.PackID + err = repo.ListBlobs(ctx, func(pb restic.PackBlob) { + h := pb.Handle() + if !done && h.Type == restic.TreeBlob { + packID = pb.PackID() done = true } }) @@ -236,12 +237,12 @@ func TestFindPackID(t *testing.T) { // load Index rtest.OK(t, repo.LoadIndex(ctx, nil)) // go through all index entries and collect data and tree packfile(s) - rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - switch blob.Type { + rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) { + switch blob.Handle().Type { case restic.DataBlob: - dataPackID = blob.PackID + dataPackID = blob.PackID() case restic.TreeBlob: - treePackID = blob.PackID + treePackID = blob.PackID() } })) return nil diff --git a/cmd/restic/cmd_list_integration_test.go b/cmd/restic/cmd_list_integration_test.go index 6b2358d47..9e5cf666c 100644 --- a/cmd/restic/cmd_list_integration_test.go +++ b/cmd/restic/cmd_list_integration_test.go @@ -71,8 +71,8 @@ func testListBlobs(t testing.TB, gopts global.Options) (blobSetFromIndex restic. // get blobs from index blobSetFromIndex = restic.NewIDSet() - rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - blobSetFromIndex.Insert(blob.ID) + rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) { + blobSetFromIndex.Insert(blob.Handle().ID) })) return nil }) diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 72e8f94f8..e716d80e4 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -75,9 +75,10 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err // tree. If it is not referenced, we have a root tree. trees := make(map[restic.ID]bool) - err = repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - if blob.Type == restic.TreeBlob { - trees[blob.Blob.ID] = false + err = repo.ListBlobs(ctx, func(blob restic.PackBlob) { + h := blob.Handle() + if h.Type == restic.TreeBlob { + trees[h.ID] = false } }) if err != nil { diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 83d2a2954..aada799e6 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -414,8 +414,8 @@ func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBl hist[i] = newSizeHistogram(2 * chunker.MaxSize) } - err := repo.ListBlobs(ctx, func(pb restic.PackedBlob) { - hist[pb.Type].Add(uint64(pb.Length)) + err := repo.ListBlobs(ctx, func(pb restic.PackBlob) { + hist[pb.Handle().Type].Add(uint64(pb.CiphertextLength())) }) return hist, err diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index d64447f7d..9f7b64259 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -269,9 +269,9 @@ func listTreePacks(gopts global.Options, t *testing.T) restic.IDSet { rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks = restic.NewIDSet() - return r.ListBlobs(ctx, func(pb restic.PackedBlob) { - if pb.Type == restic.TreeBlob { - treePacks.Insert(pb.PackID) + return r.ListBlobs(ctx, func(pb restic.PackBlob) { + if pb.Handle().Type == restic.TreeBlob { + treePacks.Insert(pb.PackID()) } }) }) @@ -318,9 +318,9 @@ func removePacksExcept(gopts global.Options, t testing.TB, keep restic.IDSet, re rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks := restic.NewIDSet() - rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) { - if pb.Type == restic.TreeBlob { - treePacks.Insert(pb.PackID) + rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackBlob) { + if pb.Handle().Type == restic.TreeBlob { + treePacks.Insert(pb.PackID()) } })) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 522729a16..2e9941dd6 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -277,8 +277,8 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er ctx, cancel := context.WithCancel(ctx) defer cancel() - err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} + err = c.repo.ListBlobs(ctx, func(blob restic.PackBlob) { + h := blob.Handle() if !c.blobRefs.M.Has(h) { debug.Log("blob %v not referenced", h) blobs = append(blobs, h) diff --git a/internal/repository/checker.go b/internal/repository/checker.go index 845048b4a..3a6986192 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -87,16 +87,18 @@ func NewChecker(repo *Repository) *Checker { } func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) { packs := make(map[restic.ID]restic.BlobType) - err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) { - tpe, exists := packs[pb.PackID] + err := idx.ListBlobs(ctx, func(pb restic.PackBlob) { + packID := pb.PackID() + h := pb.Handle() + tpe, exists := packs[packID] if exists { - if pb.Type != tpe { + if h.Type != tpe { tpe = restic.InvalidBlob } } else { - tpe = pb.Type + tpe = h.Type } - packs[pb.PackID] = tpe + packs[packID] = tpe }) return packs, err } diff --git a/internal/repository/index_list_test.go b/internal/repository/index_list_test.go index 2a9052879..9b7555709 100644 --- a/internal/repository/index_list_test.go +++ b/internal/repository/index_list_test.go @@ -26,8 +26,8 @@ func TestAllIndexBlobs(t *testing.T) { rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) fromMaster := restic.NewBlobSet() - rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - fromMaster.Insert(pb.BlobHandle) + rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + fromMaster.Insert(pb.Handle()) })) rtest.Equals(t, want, fromMaster) diff --git a/internal/repository/index_testutil_test.go b/internal/repository/index_testutil_test.go new file mode 100644 index 000000000..cc44774c4 --- /dev/null +++ b/internal/repository/index_testutil_test.go @@ -0,0 +1,17 @@ +package repository + +import ( + "github.com/restic/restic/internal/restic" +) + +// BlobsInPack returns index entries for blobs stored in packID, sorted by offset. +func BlobsInPack(repo *Repository, packID restic.ID) restic.Blobs { + var blobs restic.Blobs + for pb := range repo.idx.Values() { + if pb.PackID.Equal(packID) { + blobs = append(blobs, pb.Blob) + } + } + blobs.Sort() + return blobs +} diff --git a/internal/repository/pack/pack.go b/internal/repository/pack/pack.go index c065f4f65..25645bddd 100644 --- a/internal/repository/pack/pack.go +++ b/internal/repository/pack/pack.go @@ -65,7 +65,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL p.bytes += uint(n) p.blobs = append(p.blobs, c) - return n + CalculateEntrySize(c), nil + return n + CalculateEntrySize(c.IsCompressed()), nil } var entrySize = uint(binary.Size(restic.BlobType(0)) + 2*headerLengthSize + len(restic.ID{})) @@ -420,8 +420,8 @@ func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) { return b, size, nil } -func CalculateEntrySize(blob restic.Blob) int { - if blob.UncompressedLength != 0 { +func CalculateEntrySize(compressed bool) int { + if compressed { return int(entrySize) } return int(plainEntrySize) @@ -430,7 +430,7 @@ func CalculateEntrySize(blob restic.Blob) int { func CalculateHeaderSize(blobs restic.Blobs) int { size := headerSize for _, blob := range blobs { - size += CalculateEntrySize(blob) + size += CalculateEntrySize(blob.IsCompressed()) } return size } @@ -442,15 +442,16 @@ func CalculateHeaderSize(blobs restic.Blobs) int { func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { packSize := make(map[restic.ID]int64) - err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) { - size, ok := packSize[blob.PackID] + err := mi.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + size, ok := packSize[packID] if !ok { size = headerSize } if !onlyHdr { - size += int64(blob.Length) + size += int64(blob.CiphertextLength()) } - packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob)) + packSize[packID] = size + int64(CalculateEntrySize(blob.IsCompressed())) }) return packSize, err diff --git a/internal/repository/prune.go b/internal/repository/prune.go index d534d20e3..c7ecf3da2 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -142,11 +142,12 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed if len(plan.repackPacks) != 0 { // when repacking, we do not want to keep blobs which are // already contained in kept packs, so delete them from keepBlobs - err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) { + err := repo.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + if plan.removePacks.Has(packID) || plan.repackPacks.Has(packID) { return } - keepBlobs.Delete(blob.BlobHandle) + keepBlobs.Delete(blob.Handle()) }) if err != nil { return nil, err @@ -179,8 +180,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i // iterate over all blobs in index to find out which blobs are duplicates // The counter in usedBlobs describes how many instances of the blob exist in the repository index // Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist - err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle + err := idx.ListBlobs(ctx, func(blob restic.PackBlob) { + bh := blob.Handle() count, ok := usedBlobs.Get(bh) if ok { if count < math.MaxUint8 { @@ -229,21 +230,24 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i hasDuplicates := false // iterate over all blobs in index to generate packInfo - err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - ip := indexPack[blob.PackID] + err = idx.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + h := blob.Handle() + + ip := indexPack[packID] // Set blob type if not yet set if ip.tpe == restic.NumBlobTypes { - ip.tpe = blob.Type + ip.tpe = h.Type } // mark mixed packs with "Invalid blob type" - if ip.tpe != blob.Type { + if ip.tpe != h.Type { ip.tpe = restic.InvalidBlob } - bh := blob.BlobHandle - size := uint64(blob.Length) + bh := h + size := uint64(blob.CiphertextLength()) dupCount, _ := usedBlobs.Get(bh) switch { case dupCount >= 2: @@ -273,7 +277,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i ip.uncompressed = true } // update indexPack - indexPack[blob.PackID] = ip + indexPack[packID] = ip }) if err != nil { return nil, nil, err @@ -287,8 +291,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i // - if there are no used blobs in a pack, possibly mark duplicates as "unused" if hasDuplicates { // iterate again over all blobs in index (this is pretty cheap, all in-mem) - err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle + err = idx.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + bh := blob.Handle() count, ok := usedBlobs.Get(bh) // skip non-duplicate, aka. normal blobs // count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining @@ -296,8 +301,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i return } - ip := indexPack[blob.PackID] - size := uint64(blob.Length) + ip := indexPack[packID] + size := uint64(blob.CiphertextLength()) switch { case ip.usedBlobs > 0, ip.duplicateBlobs == ip.unusedBlobs, count == 0: // other used blobs in pack, only duplicate blobs or "last" occurrence -> transition to used @@ -325,7 +330,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i usedBlobs.Set(bh, count) } // update indexPack - indexPack[blob.PackID] = ip + indexPack[packID] = ip }) if err != nil { return nil, nil, err diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index c44839860..cbf71bf16 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -16,8 +16,8 @@ import ( func listBlobs(repo restic.Repository) restic.BlobSet { blobs := restic.NewBlobSet() - _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - blobs.Insert(pb.BlobHandle) + _ = repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + blobs.Insert(pb.Handle()) }) return blobs } @@ -66,11 +66,12 @@ func testRepairBrokenPack(t *testing.T, version uint) { // find blob that starts at offset 0 var damagedBlob restic.BlobHandle - _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - if pb.PackID == damagedID && pb.Offset == 0 { - damagedBlob = pb.BlobHandle + for _, blob := range repository.BlobsInPack(repo, damagedID) { + if blob.Offset == 0 { + damagedBlob = blob.BlobHandle + break } - }) + } return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob) }, @@ -87,11 +88,11 @@ func testRepairBrokenPack(t *testing.T, version uint) { // all blobs in the file are broken damagedBlobs := restic.NewBlobSet() - _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - if pb.PackID == damagedID { - damagedBlobs.Insert(pb.BlobHandle) + rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + if pb.PackID().Equal(damagedID) { + damagedBlobs.Insert(pb.Handle()) } - }) + })) return restic.NewIDSet(damagedID), damagedBlobs }, }, { diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 36fa0e22e..c9165a9dd 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -683,12 +683,12 @@ func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bo // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. -func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error { +func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) error { for blob := range r.idx.Values() { if ctx.Err() != nil { return ctx.Err() } - fn(blob) + fn(restic.AsPackBlob(blob)) } return nil } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 7986e2f74..1fe567b01 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -30,7 +30,7 @@ type Repository interface { NewAssociatedBlobSet() AssociatedBlobSet // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. - ListBlobs(ctx context.Context, fn func(PackedBlob)) error + ListBlobs(ctx context.Context, fn func(PackBlob)) error // ListPackHandles returns the blob handles stored in the pack file header. ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error) @@ -152,7 +152,7 @@ type Unpacked[FT FileTypes] interface { } type ListBlobser interface { - ListBlobs(ctx context.Context, fn func(PackedBlob)) error + ListBlobs(ctx context.Context, fn func(PackBlob)) error } type BlobLoader interface { From 35af104749510a961f782f8b093345bf51ee85a5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 Jun 2026 22:04:11 +0200 Subject: [PATCH 7/8] restic: change LookupBlob to return []PackBlob --- cmd/restic/cmd_copy.go | 6 +-- cmd/restic/cmd_find.go | 6 +-- cmd/restic/cmd_stats.go | 10 ++-- internal/checker/checker.go | 2 +- internal/repository/checker.go | 4 +- internal/repository/repack_test.go | 6 +-- internal/repository/repository.go | 9 +++- internal/repository/repository_test.go | 11 ++-- internal/restic/repository.go | 2 +- internal/restorer/filerestorer.go | 29 +++++----- internal/restorer/filerestorer_test.go | 74 ++++++++++++++++++-------- 11 files changed, 98 insertions(+), 61 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 28c0a2dfb..bf6e91f41 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -268,7 +268,7 @@ func copyTree(ctx context.Context, srcRepo *repository.Repository, dstRepo resti pb := srcRepo.LookupBlob(h.Type, h.ID) copyBlobs.Insert(h) for _, p := range pb { - packList.Insert(p.PackID) + packList.Insert(p.PackID()) } } } @@ -317,9 +317,9 @@ func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, pa countBlobs := 0 sizeBlobs := uint64(0) for blob := range copyBlobs.Keys() { - for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) { + for _, pb := range srcRepo.LookupBlob(blob.Type, blob.ID) { countBlobs++ - sizeBlobs += uint64(blob.Length) + sizeBlobs += uint64(pb.CiphertextLength()) break } } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index df6363d21..d669fe333 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -578,9 +578,9 @@ func (f *Finder) findObjectPack(id string, t restic.BlobType) { } for _, b := range blobs { - if b.ID.Equal(rid) { - f.printer.S("Object belongs to pack %s", b.PackID) - f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String()) + if b.Handle().ID.Equal(rid) { + f.printer.S("Object belongs to pack %s", b.PackID()) + f.printer.S(" ... Pack %s: %v", b.PackID().String(), b.Handle()) break } } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index aada799e6..f0b9fbd75 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -160,16 +160,16 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) } - stats.TotalSize += uint64(pbs[0].Length) + stats.TotalSize += uint64(pbs[0].CiphertextLength()) if repo.Config().Version >= 2 { - stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength()))) if pbs[0].IsCompressed() { - stats.TotalCompressedBlobsSize += uint64(pbs[0].Length) - stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + stats.TotalCompressedBlobsSize += uint64(pbs[0].CiphertextLength()) + stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength()))) } } stats.TotalBlobCount++ - statsProgress.Update(0, 1, uint64(pbs[0].Length)) + statsProgress.Update(0, 1, uint64(pbs[0].CiphertextLength())) } if stats.TotalCompressedBlobsSize > 0 { stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 2e9941dd6..1ccc57199 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -308,7 +308,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID // convert used blobs into their encompassing packfiles for bh := range c.blobRefs.M.Keys() { for _, pb := range c.repo.LookupBlob(bh.Type, bh.ID) { - filteredPacks[pb.PackID] = allPacks[pb.PackID] + filteredPacks[pb.PackID()] = allPacks[pb.PackID()] } } diff --git a/internal/repository/checker.go b/internal/repository/checker.go index 3a6986192..0e44ef80a 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -464,8 +464,8 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest for _, blob := range blobs { // Check if blob is contained in index and position is correct idxHas := false - for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) { - if pb.PackID == id && pb.Blob == blob { + for _, pb := range r.idx.Lookup(blob.BlobHandle) { + if pb.PackID.Equal(id) && pb.Blob == blob { idxHas = true break } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 4ead80153..909d0af91 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -141,7 +141,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe } for _, pb := range list { - packs.Insert(pb.PackID) + packs.Insert(pb.PackID()) } } @@ -221,8 +221,8 @@ func testRepack(t *testing.T, version uint) { pb := list[0] - if removePacks.Has(pb.PackID) { - t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID) + if removePacks.Has(pb.PackID()) { + t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID()) } } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index c9165a9dd..e0cfb24f8 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -672,8 +672,13 @@ func (r *Repository) Connections() uint { return r.be.Properties().Connections } -func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackedBlob { - return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) +func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob { + entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) + out := make([]restic.PackBlob, len(entries)) + for i, pb := range entries { + out[i] = restic.AsPackBlob(pb) + } + return out } // LookupBlobSize returns the size of blob id. Also returns pending blobs. diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index a35bd36d7..13320ddf7 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -233,7 +233,7 @@ func TestLoadBlobBroken(t *testing.T) { data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil) rtest.OK(t, err) rtest.Assert(t, bytes.Equal(buf, data), "data mismatch") - pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID() rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached") } @@ -422,11 +422,12 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { rtest.OK(t, err) for pb := range idx.Values() { - if _, ok := packEntries[pb.PackID]; !ok { - packEntries[pb.PackID] = make(map[restic.ID]struct{}) + packID := pb.PackID + if _, ok := packEntries[packID]; !ok { + packEntries[packID] = make(map[restic.ID]struct{}) } - packEntries[pb.PackID][id] = struct{}{} + packEntries[packID][id] = struct{}{} } return nil }) @@ -467,7 +468,7 @@ func TestListPack(t *testing.T) { repo.UseCache(c, t.Logf) // Forcibly cache pack file - packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID() rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil })) // Get size to list pack diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 1fe567b01..db969a41a 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -24,7 +24,7 @@ type Repository interface { LoadIndex(ctx context.Context, p TerminalCounterFactory) error - LookupBlob(t BlobType, id ID) []PackedBlob + LookupBlob(t BlobType, id ID) []PackBlob LookupBlobSize(t BlobType, id ID) (size uint, exists bool) NewAssociatedBlobSet() AssociatedBlobSet diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 5207f4f8d..3ec312a2c 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -47,7 +47,7 @@ type startWarmupFn func(context.Context, restic.IDSet) (restic.WarmupJob, error) // fileRestorer restores set of files type fileRestorer struct { - idx func(restic.BlobType, restic.ID) []restic.PackedBlob + idx func(restic.BlobType, restic.ID) []restic.PackBlob blobsLoader blobsLoaderFn startWarmup startWarmupFn @@ -68,7 +68,7 @@ type fileRestorer struct { func newFileRestorer(dst string, blobsLoader blobsLoaderFn, - idx func(restic.BlobType, restic.ID) []restic.PackedBlob, + idx func(restic.BlobType, restic.ID) []restic.PackBlob, connections uint, sparse bool, allowRecursiveDelete bool, @@ -102,7 +102,7 @@ func (r *fileRestorer) targetPath(location string) string { return filepath.Join(r.dst, location) } -func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int, fileOffset int64)) error { +func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(blob restic.PackBlob, idx int, fileOffset int64)) error { if len(blobIDs) == 0 { return nil } @@ -114,8 +114,8 @@ func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID return errors.Errorf("Unknown blob %s", blobID.String()) } pb := packs[0] - fn(pb.PackID, pb.Blob, i, fileOffset) - fileOffset += int64(pb.DataLength()) + fn(pb, i, fileOffset) + fileOffset += int64(pb.PlaintextLength()) } return nil @@ -143,14 +143,15 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { file.blobs = packsMap } restoredBlobs := false - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { + err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) { + packID := blob.PackID() if !file.state.HasMatchingBlob(idx) { if largeFile { - packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset}) + packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.Handle().ID, offset: fileOffset}) } restoredBlobs = true } else { - r.reportBlobProgress(file, uint64(blob.DataLength())) + r.reportBlobProgress(file, uint64(blob.PlaintextLength())) // completely ignore blob return } @@ -164,7 +165,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { packOrder = append(packOrder, packID) } pack.files[file] = struct{}{} - if blob.ID.Equal(r.zeroChunk) { + if blob.Handle().ID.Equal(r.zeroChunk) { file.sparse = r.sparse } }) @@ -278,9 +279,9 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { blobInfo.files[file] = append(blobInfo.files[file], fileOffset) } if fileBlobs, ok := file.blobs.(restic.IDs); ok { - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { - if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) { - addBlob(blob.BlobHandle, fileOffset) + err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) { + if blob.PackID().Equal(pack.id) && !file.state.HasMatchingBlob(idx) { + addBlob(blob.Handle(), fileOffset) } }) if err != nil { @@ -291,8 +292,8 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { for _, blob := range packsMap[pack.id] { idxPacks := r.idx(restic.DataBlob, blob.id) for _, idxPack := range idxPacks { - if idxPack.PackID.Equal(pack.id) { - addBlob(idxPack.BlobHandle, blob.offset) + if idxPack.PackID().Equal(pack.id) { + addBlob(idxPack.Handle(), blob.offset) break } } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index f5220998d..d517a741d 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -30,11 +30,32 @@ type TestWarmupJob struct { waitCalled bool } +type testPackBlob struct { + packID restic.ID + handle restic.BlobHandle + offset uint + ciphertext uint + plaintext uint + compressed bool +} + +var _ restic.PackBlob = (*testPackBlob)(nil) + +func (pb *testPackBlob) PackID() restic.ID { return pb.packID } + +func (pb *testPackBlob) Handle() restic.BlobHandle { return pb.handle } + +func (pb *testPackBlob) CiphertextLength() uint { return pb.ciphertext } + +func (pb *testPackBlob) PlaintextLength() uint { return pb.plaintext } + +func (pb *testPackBlob) IsCompressed() bool { return pb.compressed } + type TestRepo struct { packsIDToData map[restic.ID][]byte // blobs and files - blobs map[restic.ID][]restic.PackedBlob + blobs map[restic.ID][]restic.PackBlob files []*fileInfo filesPathToContent map[string]string @@ -44,7 +65,7 @@ type TestRepo struct { loader blobsLoaderFn } -func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackedBlob { +func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackBlob { packs := i.blobs[id] return packs } @@ -69,10 +90,16 @@ func (job *TestWarmupJob) Wait(_ context.Context) error { } func newTestRepo(content []TestFile) *TestRepo { + type packBlobLayout struct { + offset uint + ciphertext uint + plaintext uint + compressed bool + } type Pack struct { name string data []byte - blobs map[restic.ID]restic.Blob + blobs map[restic.ID]packBlobLayout } packs := make(map[string]Pack) filesPathToContent := make(map[string]string) @@ -86,21 +113,19 @@ func newTestRepo(content []TestFile) *TestRepo { var pack Pack var found bool if pack, found = packs[blob.pack]; !found { - pack = Pack{name: blob.pack, blobs: make(map[restic.ID]restic.Blob)} + pack = Pack{name: blob.pack, blobs: make(map[restic.ID]packBlobLayout)} } // calculate blob id and add to the pack as necessary blobID := restic.Hash([]byte(blob.data)) if _, found := pack.blobs[blobID]; !found { blobData := []byte(blob.data) - pack.blobs[blobID] = restic.Blob{ - BlobHandle: restic.BlobHandle{ - Type: restic.DataBlob, - ID: blobID, - }, - Length: uint(len(blobData)), - UncompressedLength: uint(len(blobData)), - Offset: uint(len(pack.data)), + n := uint(len(blobData)) + pack.blobs[blobID] = packBlobLayout{ + offset: uint(len(pack.data)), + ciphertext: n, + plaintext: n, + compressed: true, } pack.data = append(pack.data, blobData...) } @@ -110,14 +135,19 @@ func newTestRepo(content []TestFile) *TestRepo { filesPathToContent[file.name] = content } - blobs := make(map[restic.ID][]restic.PackedBlob) + blobs := make(map[restic.ID][]restic.PackBlob) packsIDToData := make(map[restic.ID][]byte) for _, pack := range packs { packID := restic.Hash(pack.data) packsIDToData[packID] = pack.data - for blobID, blob := range pack.blobs { - blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID}) + for blobID, layout := range pack.blobs { + blobs[blobID] = append(blobs[blobID], &testPackBlob{ + packID: packID, + handle: restic.BlobHandle{Type: restic.DataBlob, ID: blobID}, + offset: layout.offset, ciphertext: layout.ciphertext, + plaintext: layout.plaintext, compressed: layout.compressed, + }) } } @@ -138,12 +168,12 @@ func newTestRepo(content []TestFile) *TestRepo { warmupJobs: []*TestWarmupJob{}, } repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { - entries := make([]restic.PackedBlob, 0, len(handles)) + entries := make([]*testPackBlob, 0, len(handles)) for _, h := range handles { found := false for _, e := range repo.blobs[h.ID] { - if packID == e.PackID { - entries = append(entries, e) + if packID == e.PackID() { + entries = append(entries, e.(*testPackBlob)) found = true break } @@ -152,13 +182,13 @@ func newTestRepo(content []TestFile) *TestRepo { return fmt.Errorf("missing blob: %v", h) } } - slices.SortFunc(entries, func(a, b restic.PackedBlob) int { - return cmp.Compare(a.Offset, b.Offset) + slices.SortFunc(entries, func(a, b *testPackBlob) int { + return cmp.Compare(a.offset, b.offset) }) for _, e := range entries { - buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] - err := handleBlobFn(e.BlobHandle, buf, nil) + buf := repo.packsIDToData[packID][e.offset : e.offset+e.ciphertext] + err := handleBlobFn(e.handle, buf, nil) if err != nil { return err } From c062a78dcda421e431b6fc2afd7c42b3f521f1bb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 Jun 2026 22:21:53 +0200 Subject: [PATCH 8/8] repository: move Blob, Blobs and PackedBlob to pack package This removes them from the public interface. The latter now only provides the PackBlob interface, without being bound to the type used internally by the pack package. --- internal/repository/checker.go | 15 +- internal/repository/checker_test.go | 4 +- internal/repository/debug.go | 6 +- internal/repository/index/associated_data.go | 7 +- .../repository/index/associated_data_test.go | 25 +-- internal/repository/index/index.go | 43 ++-- .../repository/index/index_internal_test.go | 4 +- internal/repository/index/index_test.go | 87 +++++---- internal/repository/index/master_index.go | 25 +-- .../repository/index/master_index_test.go | 184 +++++++++--------- internal/repository/index_list.go | 2 +- internal/repository/index_testutil_test.go | 7 +- internal/repository/pack/blob.go | 32 +++ internal/repository/pack/blobs.go | 15 ++ internal/repository/pack/blobs_test.go | 24 +++ internal/repository/pack/pack.go | 22 +-- .../repository/pack/pack_internal_test.go | 2 +- internal/repository/pack/packedblob.go | 19 ++ internal/repository/prune.go | 3 +- internal/repository/repack.go | 8 +- internal/repository/repair_pack.go | 7 +- internal/repository/repository.go | 56 +++--- .../repository/repository_internal_test.go | 37 ++-- internal/repository/repository_test.go | 2 +- internal/restic/blob.go | 58 ------ internal/restic/blob_test.go | 19 -- internal/restic/repository.go | 5 - 27 files changed, 373 insertions(+), 345 deletions(-) create mode 100644 internal/repository/pack/blob.go create mode 100644 internal/repository/pack/blobs.go create mode 100644 internal/repository/pack/blobs_test.go create mode 100644 internal/repository/pack/packedblob.go diff --git a/internal/repository/checker.go b/internal/repository/checker.go index 0e44ef80a..351dbc247 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -129,10 +129,11 @@ func (c *Checker) LoadIndex(ctx context.Context, p restic.TerminalCounterFactory } cnt++ - if _, ok := packToIndex[blob.PackID]; !ok { - packToIndex[blob.PackID] = restic.NewIDSet() + packID := blob.PackID() + if _, ok := packToIndex[packID]; !ok { + packToIndex[packID] = restic.NewIDSet() } - packToIndex[blob.PackID].Insert(id) + packToIndex[packID].Insert(id) } for pbs := range idx.EachByPack(ctx, restic.NewIDSet()) { @@ -259,7 +260,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID type checkTask struct { id restic.ID size int64 - blobs restic.Blobs + blobs pack.Blobs } ch := make(chan checkTask) @@ -329,7 +330,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID } // checkPack reads a pack and checks the integrity of all blobs. -func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { +func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec) if err != nil { if r.cache != nil { @@ -348,7 +349,7 @@ func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Bl return err } -func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { +func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { type partialReadError struct { error @@ -465,7 +466,7 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest // Check if blob is contained in index and position is correct idxHas := false for _, pb := range r.idx.Lookup(blob.BlobHandle) { - if pb.PackID.Equal(id) && pb.Blob == blob { + if pb.PackID().Equal(id) && pb.Blob == blob { idxHas = true break } diff --git a/internal/repository/checker_test.go b/internal/repository/checker_test.go index e11584192..c1a71408c 100644 --- a/internal/repository/checker_test.go +++ b/internal/repository/checker_test.go @@ -21,7 +21,7 @@ import ( var checkerTestData = filepath.Join("..", "checker", "testdata", "checker-test-repo.tar.gz") func testWrapCheckPack(ctx context.Context, t *testing.T, repo *Repository, - packID restic.ID, blobs []restic.Blob, size int64, + packID restic.ID, blobs pack.Blobs, size int64, ) error { t.Helper() bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize) @@ -46,7 +46,7 @@ func TestGapInBlobs(t *testing.T) { _, ok := repoPacks[packID] rtest.Assert(t, ok, "expected pack 19a731a515618ec8b75fc0ff3b887d8feb83aef1001c9899f6702761142ed068") - blobs := []restic.Blob{} + blobs := pack.Blobs{} pb := <-repo.listPacksFromIndex(context.TODO(), restic.NewIDSet(packID)) blobs = append(blobs, pb.Blobs...) diff --git a/internal/repository/debug.go b/internal/repository/debug.go index 8bf25a715..dc0a81ce2 100644 --- a/internal/repository/debug.go +++ b/internal/repository/debug.go @@ -123,7 +123,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami checkPackSize(b.Blobs, len(buf), printer) - err = loadBlobs(ctx, opts, repo, id, b.Blobs, printer) + err := loadBlobs(ctx, opts, repo, id, b.Blobs, printer) if err != nil { printer.E("error: %v", err) } else { @@ -146,7 +146,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami return nil } -func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { +func checkPackSize(blobs pack.Blobs, fileSize int, printer progress.Printer) { // track current size and offset var size, offset uint64 @@ -284,7 +284,7 @@ func decryptUnsigned(k *crypto.Key, buf []byte) []byte { return out } -func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { +func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list pack.Blobs, printer progress.Printer) error { dec, err := zstd.NewReader(nil) if err != nil { panic(err) diff --git a/internal/repository/index/associated_data.go b/internal/repository/index/associated_data.go index 1267bdc4f..cd66f5c4e 100644 --- a/internal/repository/index/associated_data.go +++ b/internal/repository/index/associated_data.go @@ -159,14 +159,15 @@ func (a *AssociatedSet[T]) All() iter.Seq2[restic.BlobHandle, T] { } for pb := range a.idx.Values() { - if _, ok := a.overflow[pb.BlobHandle]; ok { + bh := pb.Handle() + if _, ok := a.overflow[bh]; ok { // already reported via overflow set continue } - val, known := a.Get(pb.BlobHandle) + val, known := a.Get(bh) if known { - if !yield(pb.BlobHandle, val) { + if !yield(bh, val) { return } } diff --git a/internal/repository/index/associated_data_test.go b/internal/repository/index/associated_data_test.go index 07e0a3d58..413f60990 100644 --- a/internal/repository/index/associated_data_test.go +++ b/internal/repository/index/associated_data_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -19,17 +20,17 @@ func (n *noopSaver) SaveUnpacked(_ context.Context, _ restic.FileType, buf []byt return restic.Hash(buf), nil } -func makeFakePackedBlob() (restic.BlobHandle, restic.PackedBlob) { +func makeFakePackedBlob() (restic.BlobHandle, *pack.PackedBlob) { bh := restic.NewRandomBlobHandle() - blob := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bh, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - return bh, blob + return bh, pb } func list(bs *AssociatedSet[uint8]) restic.BlobHandles { @@ -40,7 +41,7 @@ func TestAssociatedSet(t *testing.T) { bh, blob := makeFakePackedBlob() mi := NewMasterIndex() - test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) @@ -123,14 +124,14 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) { _, blob := makeFakePackedBlob() mi := NewMasterIndex() - test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) // add new blobs to index after building the set of, blob2 := makeFakePackedBlob() - test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) // non-existent @@ -167,10 +168,10 @@ func TestAssociatedSetIntersectAndSub(t *testing.T) { bh3, blob3 := makeFakePackedBlob() bh4, blob4 := makeFakePackedBlob() - test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, restic.Blobs{blob3.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, restic.Blobs{blob4.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob3.PackID(), pack.Blobs{blob3.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob4.PackID(), pack.Blobs{blob4.Blob}, saver)) test.OK(t, mi.Flush(context.TODO(), saver)) t.Run("Intersect", func(t *testing.T) { diff --git a/internal/repository/index/index.go b/internal/repository/index/index.go index abd8eda54..9d205ba4b 100644 --- a/internal/repository/index/index.go +++ b/internal/repository/index/index.go @@ -73,7 +73,7 @@ func (idx *Index) addToPacks(id restic.ID) int { return len(idx.packs) - 1 } -func (idx *Index) store(packIndex int, blob restic.Blob) { +func (idx *Index) store(packIndex int, blob pack.Blob) { // assert that offset and length fit into uint32! if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 { panic("offset or length does not fit in uint32. You have packs > 4GB!") @@ -146,7 +146,7 @@ func (idx *Index) Preallocate(t restic.BlobType, numEntries int) { // StorePack remembers the ids of all blobs of a given pack // in the index -func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) { +func (idx *Index) StorePack(id restic.ID, blobs pack.Blobs) { idx.m.Lock() defer idx.m.Unlock() @@ -162,23 +162,24 @@ func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) { } } -func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) restic.PackedBlob { - return restic.PackedBlob{ - Blob: restic.Blob{ +func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) *pack.PackedBlob { + return &pack.PackedBlob{ + Pack: idx.packs[e.packIndex], + Blob: pack.Blob{ BlobHandle: restic.BlobHandle{ ID: e.id, - Type: t}, + Type: t, + }, Length: uint(e.length), Offset: uint(e.offset), UncompressedLength: uint(e.uncompressedLength), }, - PackID: idx.packs[e.packIndex], } } // Lookup queries the index for the blob ID and returns all entries including -// duplicates. Adds found entries to blobs and returns the result. -func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic.PackedBlob { +// duplicates. Adds found entries to pbs and returns the result. +func (idx *Index) Lookup(bh restic.BlobHandle, pbs []*pack.PackedBlob) []*pack.PackedBlob { idx.m.RLock() defer idx.m.RUnlock() @@ -215,8 +216,8 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found // Values returns an iterator over all blobs known to the index. This blocks any // modification of the index. -func (idx *Index) Values() iter.Seq[restic.PackedBlob] { - return func(yield func(restic.PackedBlob) bool) { +func (idx *Index) Values() iter.Seq[*pack.PackedBlob] { + return func(yield func(*pack.PackedBlob) bool) { idx.m.RLock() defer idx.m.RUnlock() @@ -231,17 +232,23 @@ func (idx *Index) Values() iter.Seq[restic.PackedBlob] { } } -// EachByPack returns a channel that yields all blobs known to the index +// PackBlobs lists all blobs contained in a pack file according to the index. +type PackBlobs struct { + PackID restic.ID + Blobs pack.Blobs +} + +// EachByPack returns a channel that yields all blobs known to the index, // grouped by packID but ignoring blobs with a packID in packPlacklist for // finalized indexes. // This filtering is used when rebuilding the index where we need to ignore packs // from the finalized index which have been re-read into a non-finalized index. // When the context is cancelled, the background goroutine // terminates. This blocks any modification of the index. -func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan restic.PackBlobs { +func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan PackBlobs { idx.m.RLock() - ch := make(chan restic.PackBlobs) + ch := make(chan PackBlobs) go func() { defer idx.m.RUnlock() @@ -262,7 +269,7 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <- } for packID, packByType := range byPack { - var result restic.PackBlobs + var result PackBlobs result.PackID = packID for typ, p := range packByType { for _, e := range p { @@ -482,7 +489,7 @@ func (idx *Index) merge(idx2 *Index) error { for e := range m.valuesWithID(e2.id) { b := idx.toPackedBlob(e, restic.BlobType(typ)) b2 := idx2.toPackedBlob(e2, restic.BlobType(typ)) - if b == b2 { + if *b == *b2 { found = true break } @@ -519,7 +526,7 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, err error) { packID := idx.addToPacks(p.ID) for _, blob := range p.Blobs { - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: blob.Type, ID: blob.ID}, @@ -550,7 +557,7 @@ func (idx *Index) Len(t restic.BlobType) uint { return idx.byType[t].len() } -func PackBlobsHash(pbs restic.PackBlobs) restic.ID { +func PackBlobsHash(pbs PackBlobs) restic.ID { h := sha256.New() h.Write(pbs.PackID[:]) diff --git a/internal/repository/index/index_internal_test.go b/internal/repository/index/index_internal_test.go index 67591ef8a..94a871ed3 100644 --- a/internal/repository/index/index_internal_test.go +++ b/internal/repository/index/index_internal_test.go @@ -14,7 +14,7 @@ func TestIndexOversized(t *testing.T) { // Add blobs up to indexMaxBlobs + pack.MaxHeaderEntries - 1 packID := idx.addToPacks(restic.NewRandomID()) for i := uint(0); i < indexMaxBlobs+pack.MaxHeaderEntries-1; i++ { - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), @@ -27,7 +27,7 @@ func TestIndexOversized(t *testing.T) { rtest.Assert(t, !Oversized(idx), "index should not be considered oversized") // Add one more blob to exceed the limit - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), diff --git a/internal/repository/index/index_test.go b/internal/repository/index/index_test.go index 2a53072df..d5a779da3 100644 --- a/internal/repository/index/index_test.go +++ b/internal/repository/index/index_test.go @@ -9,19 +9,20 @@ import ( "testing" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func TestIndexSerialize(t *testing.T) { - tests := []restic.PackedBlob{} + tests := []*pack.PackedBlob{} idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 20; j++ { @@ -31,14 +32,14 @@ func TestIndexSerialize(t *testing.T) { // test a mix of compressed and uncompressed packs uncompressedLength = 2 * length } - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, UncompressedLength: uncompressedLength, }, - PackID: packID, } blobs = append(blobs, pb.Blob) tests = append(tests, pb) @@ -64,17 +65,17 @@ func TestIndexSerialize(t *testing.T) { rtest.OK(t, err) for _, testBlob := range tests { - list := idx.Lookup(testBlob.BlobHandle, nil) + list := idx.Lookup(testBlob.Handle(), nil) if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list) } result := list[0] rtest.Equals(t, testBlob, result) - list2 := idx2.Lookup(testBlob.BlobHandle, nil) + list2 := idx2.Lookup(testBlob.Handle(), nil) if len(list2) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list2), list2) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list2), list2) } result2 := list2[0] @@ -82,21 +83,21 @@ func TestIndexSerialize(t *testing.T) { } // add more blobs to idx - newtests := []restic.PackedBlob{} + newtests := []*pack.PackedBlob{} for i := 0; i < 10; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 10; j++ { length := uint(i*100 + j) - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, }, - PackID: packID, } blobs = append(blobs, pb.Blob) newtests = append(newtests, pb) @@ -127,9 +128,9 @@ func TestIndexSerialize(t *testing.T) { // all new blobs must be in the index for _, testBlob := range newtests { - list := idx3.Lookup(testBlob.BlobHandle, nil) + list := idx3.Lookup(testBlob.Handle(), nil) if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list) } blob := list[0] @@ -145,12 +146,12 @@ func TestIndexSize(t *testing.T) { blobCount := 100 for i := 0; i < packs; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < blobCount; j++ { length := uint(i*100 + j) - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, @@ -293,15 +294,15 @@ func TestIndexUnserialize(t *testing.T) { t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) - rtest.Equals(t, test.packID, blob.PackID) - rtest.Equals(t, test.tpe, blob.Type) - rtest.Equals(t, test.offset, blob.Offset) - rtest.Equals(t, test.length, blob.Length) + rtest.Equals(t, test.packID, blob.PackID()) + rtest.Equals(t, test.tpe, blob.Blob.Type) + rtest.Equals(t, test.offset, blob.Blob.Offset) + rtest.Equals(t, test.length, blob.Blob.Length) switch task.version { case 1: - rtest.Equals(t, uint(0), blob.UncompressedLength) + rtest.Equals(t, uint(0), blob.Blob.UncompressedLength) case 2: - rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength) + rtest.Equals(t, test.uncompressedLength, blob.Blob.UncompressedLength) default: t.Fatal("Invalid index version") } @@ -313,20 +314,20 @@ func TestIndexUnserialize(t *testing.T) { } for _, blob := range blobs { - b, ok := exampleLookupTest.blobs[blob.ID] + b, ok := exampleLookupTest.blobs[blob.Handle().ID] if !ok { - t.Errorf("unexpected blob %v found", blob.ID.Str()) + t.Errorf("unexpected blob %v found", blob.Handle().ID.String()) } - if blob.Type != b { - t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) + if blob.Blob.Type != b { + t.Errorf("unexpected type for blob %v: want %v, got %v", blob.Handle().ID.String(), b, blob.Blob.Type) } } } } -func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) { +func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []*pack.PackedBlob) { for pb := range idx.Values() { - if pb.PackID.Equal(id) { + if pb.PackID().Equal(id) { pbs = append(pbs, pb) } } @@ -401,7 +402,7 @@ func TestIndexPacks(t *testing.T) { for i := 0; i < 20; i++ { packID := restic.NewRandomID() - idx.StorePack(packID, restic.Blobs{ + idx.StorePack(packID, pack.Blobs{ { BlobHandle: restic.NewRandomBlobHandle(), Offset: 0, @@ -433,12 +434,12 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupB // create index with given number of pack files for i := 0; i < packfiles; i++ { packID := NewRandomTestID(rng) - var blobs restic.Blobs + var blobs pack.Blobs offset := 0 for offset < maxPackSize { size := 2000 + rng.Intn(4*1024*1024) id := NewRandomTestID(rng) - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: id, @@ -482,7 +483,7 @@ func BenchmarkIndexHasKnown(b *testing.B) { idx, _ := createRandomIndex(rand.New(rand.NewSource(0)), 200000) handles := make([]restic.BlobHandle, 0, 100000) for handle := range idx.Values() { - handles = append(handles, handle.BlobHandle) + handles = append(handles, handle.Handle()) if len(handles) == cap(handles) { break } @@ -517,14 +518,14 @@ func BenchmarkIndexAllocParallel(b *testing.B) { } func TestIndexHas(t *testing.T) { - tests := []restic.PackedBlob{} + tests := []*pack.PackedBlob{} idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 20; j++ { @@ -534,14 +535,14 @@ func TestIndexHas(t *testing.T) { // test a mix of compressed and uncompressed packs uncompressedLength = 2 * length } - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, UncompressedLength: uncompressedLength, }, - PackID: packID, } blobs = append(blobs, pb.Blob) tests = append(tests, pb) @@ -551,11 +552,11 @@ func TestIndexHas(t *testing.T) { } for _, testBlob := range tests { - rtest.Assert(t, idx.Has(testBlob.BlobHandle), "Index reports not having data blob added to it") + rtest.Assert(t, idx.Has(testBlob.Handle()), "Index reports not having data blob added to it") } rtest.Assert(t, !idx.Has(restic.NewRandomBlobHandle()), "Index reports having a data blob not added to it") - rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob") + rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].Handle().ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob") } func TestMixedEachByPack(t *testing.T) { @@ -566,7 +567,7 @@ func TestMixedEachByPack(t *testing.T) { for i := 0; i < 50; i++ { packID := restic.NewRandomID() expected[packID] = 1 - blobs := restic.Blobs{ + blobs := pack.Blobs{ { BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, Offset: 0, @@ -608,7 +609,7 @@ func TestEachByPackIgnoes(t *testing.T) { } else { expected[packID] = 1 } - blobs := restic.Blobs{ + blobs := pack.Blobs{ { BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, Offset: 0, diff --git a/internal/repository/index/master_index.go b/internal/repository/index/master_index.go index 84db0d962..a667e6f4f 100644 --- a/internal/repository/index/master_index.go +++ b/internal/repository/index/master_index.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" @@ -39,10 +40,11 @@ func (mi *MasterIndex) clearPendingBlobs() { } // Lookup queries all known Indexes for the ID and returns all matches. -func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) { +func (mi *MasterIndex) Lookup(bh restic.BlobHandle) []*pack.PackedBlob { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() + var pbs []*pack.PackedBlob for _, idx := range mi.idx { pbs = idx.Lookup(bh, pbs) } @@ -145,12 +147,12 @@ func (mi *MasterIndex) Insert(idx *Index) { } // StorePack remembers the id and pack in the index. -func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs restic.Blobs, r restic.SaverUnpacked[restic.FileType]) error { +func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs pack.Blobs, r restic.SaverUnpacked[restic.FileType]) error { mi.storePack(id, blobs) return mi.saveFullIndex(ctx, r) } -func (mi *MasterIndex) storePack(id restic.ID, blobs restic.Blobs) { +func (mi *MasterIndex) storePack(id restic.ID, blobs pack.Blobs) { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() @@ -218,8 +220,8 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index { // Values returns an iterator over all blobs known to the index. This blocks any // modification of the index. -func (mi *MasterIndex) Values() iter.Seq[restic.PackedBlob] { - return func(yield func(restic.PackedBlob) bool) { +func (mi *MasterIndex) Values() iter.Seq[*pack.PackedBlob] { + return func(yield func(*pack.PackedBlob) bool) { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() @@ -663,13 +665,13 @@ func (mi *MasterIndex) saveFullIndex(ctx context.Context, r restic.SaverUnpacked } // ListPacks returns the blobs of the specified pack files grouped by pack file. -func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { - out := make(chan restic.PackBlobs) +func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan PackBlobs { + out := make(chan PackBlobs) go func() { defer close(out) // only resort a part of the index to keep the memory overhead bounded for i := byte(0); i < 16; i++ { - packBlob := make(map[restic.ID]restic.Blobs) + packBlob := make(map[restic.ID]pack.Blobs) for pack := range packs { if pack[0]&0xf == i { packBlob[pack] = nil @@ -682,8 +684,9 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan if ctx.Err() != nil { return } - if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i { - packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob) + packID := pb.PackID() + if packs.Has(packID) && packID[0]&0xf == i { + packBlob[packID] = append(packBlob[packID], pb.Blob) } } @@ -692,7 +695,7 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan // allow GC packBlob[packID] = nil select { - case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}: + case out <- PackBlobs{PackID: packID, Blobs: pbs}: case <-ctx.Done(): return } diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index 56d01a503..ed320cc3b 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/progress" @@ -24,18 +25,18 @@ func TestMasterIndex(t *testing.T) { bhInIdx2 := restic.NewRandomBlobHandle() bhInIdx12 := restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob} - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx1, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx2, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -43,9 +44,9 @@ func TestMasterIndex(t *testing.T) { }, } - blob12a := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob12a := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx12, Length: uint(crypto.CiphertextLength(123)), Offset: 110, @@ -53,9 +54,9 @@ func TestMasterIndex(t *testing.T) { }, } - blob12b := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob12b := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx12, Length: uint(crypto.CiphertextLength(123)), Offset: 50, @@ -64,12 +65,12 @@ func TestMasterIndex(t *testing.T) { } idx1 := index.NewIndex() - idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) - idx1.StorePack(blob12a.PackID, restic.Blobs{blob12a.Blob}) + idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) + idx1.StorePack(blob12a.PackID(), pack.Blobs{blob12a.Blob}) idx2 := index.NewIndex() - idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) - idx2.StorePack(blob12b.PackID, restic.Blobs{blob12b.Blob}) + idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) + idx2.StorePack(blob12b.PackID(), pack.Blobs{blob12b.Blob}) mIdx := index.NewMasterIndex() mIdx.Insert(idx1) @@ -77,7 +78,7 @@ func TestMasterIndex(t *testing.T) { // test idInIdx1 blobs := mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) size, found := mIdx.LookupSize(bhInIdx1) rtest.Equals(t, true, found) @@ -85,7 +86,7 @@ func TestMasterIndex(t *testing.T) { // test idInIdx2 blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) size, found = mIdx.LookupSize(bhInIdx2) rtest.Equals(t, true, found) @@ -95,19 +96,20 @@ func TestMasterIndex(t *testing.T) { blobs = mIdx.Lookup(bhInIdx12) rtest.Equals(t, 2, len(blobs)) - // test Lookup result for blob12a - found = false - if blobs[0] == blob12a || blobs[1] == blob12a { - found = true + containsPackedBlob := func(list []*pack.PackedBlob, want *pack.PackedBlob) bool { + for _, b := range list { + if b.PackID().Equal(want.PackID()) && b.Blob == want.Blob { + return true + } + } + return false } - rtest.Assert(t, found, "blob12a not found in result") + + // test Lookup result for blob12a + rtest.Assert(t, containsPackedBlob(blobs, blob12a), "blob12a not found in result") // test Lookup result for blob12b - found = false - if blobs[0] == blob12b || blobs[1] == blob12b { - found = true - } - rtest.Assert(t, found, "blob12a not found in result") + rtest.Assert(t, containsPackedBlob(blobs, blob12b), "blob12b not found in result") size, found = mIdx.LookupSize(bhInIdx12) rtest.Equals(t, true, found) @@ -135,7 +137,7 @@ func TestMasterIndexAddPending(t *testing.T) { // Test AddPending: try to add a blob that's already in an index (should return false) bhInIndex := restic.NewRandomBlobHandle() idx := index.NewIndex() - idx.StorePack(restic.NewRandomID(), restic.Blobs{{ + idx.StorePack(restic.NewRandomID(), pack.Blobs{{ BlobHandle: bhInIndex, Length: uint(crypto.CiphertextLength(50)), Offset: 0, @@ -173,14 +175,14 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) { // Store the blob in a pack packID := restic.NewRandomID() - blob := restic.Blob{ + blob := pack.Blob{ BlobHandle: bhPending, Length: uint(crypto.CiphertextLength(75)), Offset: 0, UncompressedLength: 75, } saver := &noopSaver{} - err := mIdx.StorePack(context.Background(), packID, restic.Blobs{blob}, saver) + err := mIdx.StorePack(context.Background(), packID, pack.Blobs{blob}, saver) rtest.OK(t, err) // Verify it is still found @@ -191,8 +193,8 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) { // Verify the blob can be found via Lookup from the index blobs := mIdx.Lookup(bhPending) rtest.Assert(t, len(blobs) > 0, "blob should be found in index after StorePack") - rtest.Equals(t, packID, blobs[0].PackID) - rtest.Equals(t, bhPending, blobs[0].BlobHandle) + rtest.Equals(t, packID, blobs[0].PackID()) + rtest.Equals(t, bhPending, blobs[0].Handle()) // Test that adding the same blob as pending again fails (it's now in index) added = mIdx.AddPending(bhPending, 100) @@ -203,18 +205,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) { bhInIdx1 := restic.NewRandomBlobHandle() bhInIdx2 := restic.NewRandomBlobHandle() - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx1, Length: 10, Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx2, Length: 100, Offset: 10, @@ -223,10 +225,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) { } idx1 := index.NewIndex() - idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) + idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) idx2 := index.NewIndex() - idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) + idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) mIdx := index.NewMasterIndex() mIdx.Insert(idx1) @@ -246,18 +248,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) { rtest.Equals(t, 2, blobCount) blobs := mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) blobs = mIdx.Lookup(restic.NewRandomBlobHandle()) rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") // merge another index containing identical blobs idx3 := index.NewIndex() - idx3.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) - idx3.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) + idx3.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) + idx3.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) mIdx.Insert(idx3) finalIndexes, idxCount, newIDs := index.TestMergeIndex(t, mIdx) @@ -268,10 +270,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) { // Index should have same entries as before! blobs = mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) blobCount = 0 for range mIdx.Values() { @@ -448,9 +450,9 @@ func testIndexSave(t *testing.T, version uint) { idx := index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) - blobs := make(map[restic.PackedBlob]struct{}) + blobs := make(map[pack.PackedBlob]struct{}) for pb := range idx.Values() { - blobs[pb] = struct{}{} + blobs[*pb] = struct{}{} } rtest.OK(t, test.saver(idx, unpacked)) @@ -458,8 +460,8 @@ func testIndexSave(t *testing.T, version uint) { rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) for pb := range idx.Values() { - if _, ok := blobs[pb]; ok { - delete(blobs, pb) + if _, ok := blobs[*pb]; ok { + delete(blobs, *pb) } else { t.Fatalf("unexpected blobs %v", pb) } @@ -481,9 +483,9 @@ func testIndexSavePartial(t *testing.T, version uint) { // capture blob list before adding fourth snapshot idx := index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) - blobs := make(map[restic.PackedBlob]struct{}) + blobs := make(map[pack.PackedBlob]struct{}) for pb := range idx.Values() { - blobs[pb] = struct{}{} + blobs[*pb] = struct{}{} } // add+remove new snapshot and track its pack files @@ -502,8 +504,8 @@ func testIndexSavePartial(t *testing.T, version uint) { idx = index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) for pb := range idx.Values() { - if _, ok := blobs[pb]; ok { - delete(blobs, pb) + if _, ok := blobs[*pb]; ok { + delete(blobs, *pb) } else { t.Fatalf("unexpected blobs %v", pb) } @@ -516,7 +518,7 @@ func testIndexSavePartial(t *testing.T, version uint) { checker.TestCheckRepo(t, repo) } -func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[restic.PackedBlob]struct{} { +func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[pack.PackedBlob]struct{} { p := progress.NewCounter(0, 0, nil) rtest.OK(t, master.Load(context.TODO(), repo, p, nil)) v, max := p.Get() @@ -525,10 +527,10 @@ func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, ma return collectBlobs(master) } -func collectBlobs(master *index.MasterIndex) map[restic.PackedBlob]struct{} { - s := make(map[restic.PackedBlob]struct{}) +func collectBlobs(master *index.MasterIndex) map[pack.PackedBlob]struct{} { + s := make(map[pack.PackedBlob]struct{}) for pb := range master.Values() { - s[pb] = struct{}{} + s[*pb] = struct{}{} } return s } @@ -588,17 +590,17 @@ func TestRewriteOversizedIndex(t *testing.T) { return idx.Len(restic.DataBlob) > 2*fullIndexCount } - var blobs restic.Blobs + var blobs pack.Blobs // build oversized index idx := index.NewIndex() numPacks := 5 for p := 0; p < numPacks; p++ { packID := restic.NewRandomID() - packBlobs := make(restic.Blobs, 0, fullIndexCount) + packBlobs := make(pack.Blobs, 0, fullIndexCount) for i := 0; i < fullIndexCount; i++ { - blob := restic.Blob{ + blob := pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), @@ -645,17 +647,17 @@ func TestRewriteSplitPacks(t *testing.T) { bh2 := restic.NewRandomBlobHandle() bhOther := restic.NewRandomBlobHandle() - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bh1, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: blob1.PackID, - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: blob1.PackID(), + Blob: pack.Blob{ BlobHandle: bh2, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -663,9 +665,9 @@ func TestRewriteSplitPacks(t *testing.T) { }, } // used to force index repacking - blobOther := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blobOther := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhOther, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -673,25 +675,25 @@ func TestRewriteSplitPacks(t *testing.T) { } mi := index.NewMasterIndex() - rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID), nil, nil, index.MasterIndexRewriteOpts{})) + rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID()), nil, nil, index.MasterIndexRewriteOpts{})) mi = index.NewMasterIndex() rtest.OK(t, mi.Load(context.TODO(), repo, nil, nil)) // test that all blobs are still in the index - for _, blob := range []restic.PackedBlob{blob1, blob2} { - blobs := mi.Lookup(blob.BlobHandle) - rtest.Equals(t, []restic.PackedBlob{blob}, blobs) + for _, blob := range []*pack.PackedBlob{blob1, blob2} { + blobs := mi.Lookup(blob.Handle()) + rtest.Equals(t, []*pack.PackedBlob{blob}, blobs) } - blobs := mi.Lookup(blobOther.BlobHandle) + blobs := mi.Lookup(blobOther.Handle()) rtest.Equals(t, nil, blobs) } @@ -713,17 +715,17 @@ func TestRewriteFullPacks(t *testing.T) { packA := restic.NewRandomID() packB := restic.NewRandomID() - blobA := restic.PackedBlob{ - PackID: packA, - Blob: restic.Blob{ + blobA := &pack.PackedBlob{ + Pack: packA, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blobB := restic.PackedBlob{ - PackID: packB, - Blob: restic.Blob{ + blobB := &pack.PackedBlob{ + Pack: packB, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Length: uint(crypto.CiphertextLength(50)), Offset: 0, @@ -731,11 +733,11 @@ func TestRewriteFullPacks(t *testing.T) { } mi := index.NewMasterIndex() - rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packA, pack.Blobs{blobA.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) indexIDs := mi.IDs() @@ -750,6 +752,6 @@ func TestRewriteFullPacks(t *testing.T) { rtest.Equals(t, 2, len(afterRewrite)) rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs))) - rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle)) - rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle)) + rtest.Equals(t, []*pack.PackedBlob{blobA}, mi2.Lookup(blobA.Handle())) + rtest.Equals(t, []*pack.PackedBlob{blobB}, mi2.Lookup(blobB.Handle())) } diff --git a/internal/repository/index_list.go b/internal/repository/index_list.go index 18b40f114..c6d003539 100644 --- a/internal/repository/index_list.go +++ b/internal/repository/index_list.go @@ -28,7 +28,7 @@ func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.Load if ctx.Err() != nil { return ctx.Err() } - if !yield(IndexBlob{Handle: blob.BlobHandle}) { + if !yield(IndexBlob{Handle: blob.Handle()}) { return stopIteration } } diff --git a/internal/repository/index_testutil_test.go b/internal/repository/index_testutil_test.go index cc44774c4..9e3a158ae 100644 --- a/internal/repository/index_testutil_test.go +++ b/internal/repository/index_testutil_test.go @@ -1,14 +1,15 @@ package repository import ( + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" ) // BlobsInPack returns index entries for blobs stored in packID, sorted by offset. -func BlobsInPack(repo *Repository, packID restic.ID) restic.Blobs { - var blobs restic.Blobs +func BlobsInPack(repo *Repository, packID restic.ID) pack.Blobs { + var blobs pack.Blobs for pb := range repo.idx.Values() { - if pb.PackID.Equal(packID) { + if pb.PackID().Equal(packID) { blobs = append(blobs, pb.Blob) } } diff --git a/internal/repository/pack/blob.go b/internal/repository/pack/blob.go new file mode 100644 index 000000000..5ec8e6680 --- /dev/null +++ b/internal/repository/pack/blob.go @@ -0,0 +1,32 @@ +package pack + +import ( + "fmt" + + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/restic" +) + +// Blob is one part of a file or a tree with pack layout information. +type Blob struct { + restic.BlobHandle + Length uint + Offset uint + UncompressedLength uint +} + +func (b Blob) String() string { + return fmt.Sprintf("", + b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength) +} + +func (b Blob) DataLength() uint { + if b.UncompressedLength != 0 { + return b.UncompressedLength + } + return uint(crypto.PlaintextLength(int(b.Length))) +} + +func (b Blob) IsCompressed() bool { + return b.UncompressedLength != 0 +} diff --git a/internal/repository/pack/blobs.go b/internal/repository/pack/blobs.go new file mode 100644 index 000000000..04fe04d84 --- /dev/null +++ b/internal/repository/pack/blobs.go @@ -0,0 +1,15 @@ +package pack + +import ( + "cmp" + "slices" +) + +// Blobs is a list of blobs with pack layout information (offset, length, ...). +type Blobs []Blob + +func (b Blobs) Sort() { + slices.SortFunc(b, func(a, b Blob) int { + return cmp.Compare(a.Offset, b.Offset) + }) +} diff --git a/internal/repository/pack/blobs_test.go b/internal/repository/pack/blobs_test.go new file mode 100644 index 000000000..b345e482f --- /dev/null +++ b/internal/repository/pack/blobs_test.go @@ -0,0 +1,24 @@ +package pack + +import ( + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestBlobsSort(t *testing.T) { + blobs := Blobs{ + {Offset: 100}, + {Offset: 0}, + {Offset: 50}, + } + blobs.Sort() + rtest.Equals(t, uint(0), blobs[0].Offset) + rtest.Equals(t, uint(50), blobs[1].Offset) + rtest.Equals(t, uint(100), blobs[2].Offset) +} + +func TestBlobsSortNilSlice(t *testing.T) { + var blobs Blobs + blobs.Sort() +} diff --git a/internal/repository/pack/pack.go b/internal/repository/pack/pack.go index 25645bddd..b23043e7f 100644 --- a/internal/repository/pack/pack.go +++ b/internal/repository/pack/pack.go @@ -21,7 +21,7 @@ var ErrBroken = errors.New("packer cannot be used after a write error") // Packer is used to create a new Pack. type Packer struct { - blobs restic.Blobs + blobs []Blob bytes uint k *crypto.Key @@ -56,7 +56,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL return n, p.err } - c := restic.Blob{ + c := Blob{ BlobHandle: restic.BlobHandle{Type: t, ID: id}, Length: uint(n), Offset: p.bytes, @@ -129,7 +129,7 @@ func (p *Packer) Finalize() error { return nil } -func verifyHeader(k *crypto.Key, header []byte, expected restic.Blobs) error { +func verifyHeader(k *crypto.Key, header []byte, expected []Blob) error { // do not offer a way to skip the pack header verification, as pack headers are usually small enough // to not result in a significant performance impact @@ -157,7 +157,7 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func makeHeader(blobs restic.Blobs) ([]byte, error) { +func makeHeader(blobs []Blob) ([]byte, error) { buf := make([]byte, 0, len(blobs)*int(entrySize)) for _, b := range blobs { @@ -232,7 +232,7 @@ func (p *Packer) HeaderFull() bool { } // Blobs returns the slice of blobs that have been written. -func (p *Packer) Blobs() restic.Blobs { +func (p *Packer) Blobs() Blobs { p.m.Lock() defer p.m.Unlock() @@ -348,7 +348,7 @@ func (e InvalidFileError) Error() string { // List returns the list of entries found in a pack file and the length of the // header (including header size and crypto overhead) -func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrSize uint32, err error) { +func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries Blobs, hdrSize uint32, err error) { buf, err := readHeader(rd, size) if err != nil { return nil, 0, err @@ -367,7 +367,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS } // might over allocate a bit if all blobs have EntrySize but only by a few percent - entries = make(restic.Blobs, 0, uint(len(buf))/plainEntrySize) + entries = make(Blobs, 0, uint(len(buf))/plainEntrySize) pos := uint(0) for len(buf) > 0 { @@ -385,7 +385,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS return entries, hdrSize, nil } -func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) { +func parseHeaderEntry(p []byte) (b Blob, size uint, err error) { l := uint(len(p)) size = plainEntrySize if l < plainEntrySize { @@ -427,7 +427,7 @@ func CalculateEntrySize(compressed bool) int { return int(plainEntrySize) } -func CalculateHeaderSize(blobs restic.Blobs) int { +func CalculateHeaderSize(blobs Blobs) int { size := headerSize for _, blob := range blobs { size += CalculateEntrySize(blob.IsCompressed()) @@ -439,10 +439,10 @@ func CalculateHeaderSize(blobs restic.Blobs) int { // If onlyHdr is set to true, only the size of the header is returned // Note that this function only gives correct sizes, if there are no // duplicates in the index. -func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { +func Size(ctx context.Context, idx restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { packSize := make(map[restic.ID]int64) - err := mi.ListBlobs(ctx, func(blob restic.PackBlob) { + err := idx.ListBlobs(ctx, func(blob restic.PackBlob) { packID := blob.PackID() size, ok := packSize[packID] if !ok { diff --git a/internal/repository/pack/pack_internal_test.go b/internal/repository/pack/pack_internal_test.go index 186450e1c..d9a15298a 100644 --- a/internal/repository/pack/pack_internal_test.go +++ b/internal/repository/pack/pack_internal_test.go @@ -182,7 +182,7 @@ func TestReadRecords(t *testing.T) { func TestUnpackedVerification(t *testing.T) { // create random keys k := crypto.NewRandomKey() - blobs := restic.Blobs{ + blobs := []Blob{ { BlobHandle: restic.NewRandomBlobHandle(), Length: 42, diff --git a/internal/repository/pack/packedblob.go b/internal/repository/pack/packedblob.go new file mode 100644 index 000000000..e5d3e3588 --- /dev/null +++ b/internal/repository/pack/packedblob.go @@ -0,0 +1,19 @@ +package pack + +import "github.com/restic/restic/internal/restic" + +// PackedBlob is one index entry for a blob in a pack (may be duplicate across indexes). +type PackedBlob struct { + Pack restic.ID + Blob Blob +} + +func (pb *PackedBlob) PackID() restic.ID { return pb.Pack } + +func (pb *PackedBlob) Handle() restic.BlobHandle { return pb.Blob.BlobHandle } + +func (pb *PackedBlob) CiphertextLength() uint { return pb.Blob.Length } + +func (pb *PackedBlob) PlaintextLength() uint { return pb.Blob.DataLength() } + +func (pb *PackedBlob) IsCompressed() bool { return pb.Blob.IsCompressed() } diff --git a/internal/repository/prune.go b/internal/repository/prune.go index c7ecf3da2..becbc99d2 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -246,9 +246,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i ip.tpe = restic.InvalidBlob } - bh := h size := uint64(blob.CiphertextLength()) - dupCount, _ := usedBlobs.Get(bh) + dupCount, _ := usedBlobs.Get(h) switch { case dupCount >= 2: hasDuplicates = true diff --git a/internal/repository/repack.go b/internal/repository/repack.go index a4472c91e..d8e351089 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -7,6 +7,8 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" @@ -80,11 +82,11 @@ func repack( } var keepMutex sync.Mutex - downloadQueue := make(chan restic.PackBlobs) + downloadQueue := make(chan index.PackBlobs) wg.Go(func() error { defer close(downloadQueue) for pbs := range repo.listPacksFromIndex(wgCtx, packs) { - var packBlobs restic.Blobs + var packBlobs pack.Blobs keepMutex.Lock() // filter out unnecessary blobs for _, entry := range pbs.Blobs { @@ -96,7 +98,7 @@ func repack( keepMutex.Unlock() select { - case downloadQueue <- restic.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}: + case downloadQueue <- index.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}: case <-wgCtx.Done(): return wgCtx.Err() } diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index a7d04bbc2..1c27acf5b 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -6,6 +6,7 @@ import ( "io" "slices" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" ) @@ -71,8 +72,8 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe return nil } -func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]restic.Blobs, error) { - packToBlobs := make(map[restic.ID]restic.Blobs) +func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]pack.Blobs, error) { + packToBlobs := make(map[restic.ID]pack.Blobs) err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { if ids.Has(id) { @@ -90,7 +91,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe return packToBlobs, nil } -func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs restic.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error { +func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs pack.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error { err := repo.loadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { printer.E("failed to load blob %v: %v", blob.ID, err) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index e0cfb24f8..828559f3b 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -213,7 +213,7 @@ type haver interface { } // sortCachedPacksFirst moves all cached pack files to the front of blobs. -func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { +func sortCachedPacksFirst(cache haver, blobs []*pack.PackedBlob) { if cache == nil { return } @@ -224,10 +224,10 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { } cached := blobs[:0] - noncached := make([]restic.PackedBlob, 0, len(blobs)/2) + noncached := make([]*pack.PackedBlob, 0, len(blobs)/2) for _, blob := range blobs { - if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { + if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID().String()}) { cached = append(cached, blob) continue } @@ -256,7 +256,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. if err != nil { if r.cache != nil { for _, blob := range blobs { - h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()} // ignore errors as there's not much we can do here _ = r.cache.Forget(h) } @@ -267,28 +267,28 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. return buf, err } -func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, buf []byte) ([]byte, error) { +func (r *Repository) loadBlob(ctx context.Context, blobs []*pack.PackedBlob, buf []byte) ([]byte, error) { var lastError error for _, blob := range blobs { - debug.Log("blob %v found: %v", blob.BlobHandle, blob) + debug.Log("blob %v found: %v", blob.Handle(), blob) // load blob from pack - h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()} switch { - case cap(buf) < int(blob.Length): - buf = make([]byte, blob.Length) - case len(buf) != int(blob.Length): - buf = buf[:blob.Length] + case cap(buf) < int(blob.Blob.Length): + buf = make([]byte, blob.Blob.Length) + case len(buf) != int(blob.Blob.Length): + buf = buf[:blob.Blob.Length] } - _, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) + _, err := backend.ReadAt(ctx, r.be, h, int64(blob.Blob.Offset), buf) if err != nil { debug.Log("error loading blob %v: %v", blob, err) lastError = err continue } - it := newPackBlobIterator(blob.PackID, newByteReader(buf), blob.Offset, restic.Blobs{blob.Blob}, r.key, r.getZstdDecoder()) + it := newPackBlobIterator(blob.PackID(), newByteReader(buf), blob.Blob.Offset, pack.Blobs{blob.Blob}, r.key, r.getZstdDecoder()) pbv, err := it.Next() if err == nil { @@ -314,7 +314,7 @@ func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, bu return nil, lastError } - return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs)) + return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].Handle(), len(blobs)) } func (r *Repository) getZstdEncoder() *zstd.Encoder { @@ -675,8 +675,8 @@ func (r *Repository) Connections() uint { func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob { entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) out := make([]restic.PackBlob, len(entries)) - for i, pb := range entries { - out[i] = restic.AsPackBlob(pb) + for i, e := range entries { + out[i] = e } return out } @@ -693,13 +693,13 @@ func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) er if ctx.Err() != nil { return ctx.Err() } - fn(restic.AsPackBlob(blob)) + fn(blob) } return nil } // listPacksFromIndex returns index entries for the given packs, grouped by pack file. -func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { +func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan index.PackBlobs { return r.idx.ListPacks(ctx, packs) } @@ -971,7 +971,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic } // listPack returns blob entries from the pack file header including offsets. -func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) { +func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (pack.Blobs, error) { h := backend.Handle{Type: restic.PackFile, Name: id.String()} entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) @@ -984,7 +984,7 @@ func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (re // retry on error entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) } - return entries, err + return pack.Blobs(entries), err } // ListPackHandles returns the blob handles stored in the pack file header. @@ -1081,12 +1081,12 @@ func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, ha return r.loadBlobsFromPack(ctx, packID, blobs, handleBlobFn) } -func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (restic.Blobs, error) { - blobs := make(restic.Blobs, 0, len(handles)) +func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (pack.Blobs, error) { + blobs := make(pack.Blobs, 0, len(handles)) for _, h := range handles { found := false for _, pb := range r.idx.Lookup(h) { - if pb.PackID.Equal(packID) { + if pb.PackID().Equal(packID) { blobs = append(blobs, pb.Blob) found = true break @@ -1099,11 +1099,11 @@ func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) return blobs, nil } -func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return streamPack(ctx, r.be.Load, r.LoadBlob, r.getZstdDecoder(), r.key, packID, blobs, handleBlobFn) } -func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do return nil @@ -1146,7 +1146,7 @@ func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn return streamPackPart(ctx, beLoad, loadBlobFn, dec, key, packID, blobs[lowerIdx:], handleBlobFn) } -func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: blobs[0].Type.IsMetadata()} dataStart := blobs[0].Offset @@ -1256,7 +1256,7 @@ type packBlobIterator struct { rd discardReader currentOffset uint - blobs restic.Blobs + blobs pack.Blobs key *crypto.Key dec *zstd.Decoder @@ -1272,7 +1272,7 @@ type packBlobValue struct { var errPackEOF = errors.New("reached EOF of pack file") func newPackBlobIterator(packID restic.ID, rd discardReader, currentOffset uint, - blobs restic.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator { + blobs pack.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator { return &packBlobIterator{ packID: packID, rd: rd, diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index 31aed6f64..226c3c237 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -17,6 +17,7 @@ import ( "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,7 +28,7 @@ func (c mapcache) Has(h backend.Handle) bool { return c[h] } func TestSortCachedPacksFirst(t *testing.T) { var ( - blobs, sorted [100]restic.PackedBlob + blobs, sorted [100]*pack.PackedBlob cache = make(mapcache) r = rand.New(rand.NewSource(1261)) @@ -36,7 +37,7 @@ func TestSortCachedPacksFirst(t *testing.T) { for i := 0; i < len(blobs); i++ { var id restic.ID r.Read(id[:]) - blobs[i] = restic.PackedBlob{PackID: id} + blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}} if i%3 == 0 { h := backend.Handle{Name: id.String(), Type: backend.PackFile} @@ -46,8 +47,8 @@ func TestSortCachedPacksFirst(t *testing.T) { copy(sorted[:], blobs[:]) sort.SliceStable(sorted[:], func(i, j int) bool { - hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()} - hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()} + hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID().String()} + hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID().String()} return cache.Has(hi) && !cache.Has(hj) }) @@ -59,7 +60,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { const nblobs = 512 // Corresponds to a file of ca. 2GB. var ( - blobs [nblobs]restic.PackedBlob + blobs [nblobs]*pack.PackedBlob cache = make(mapcache) r = rand.New(rand.NewSource(1261)) ) @@ -67,7 +68,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { for i := 0; i < nblobs; i++ { var id restic.ID r.Read(id[:]) - blobs[i] = restic.PackedBlob{PackID: id} + blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}} if i%3 == 0 { h := backend.Handle{Name: id.String(), Type: backend.PackFile} @@ -75,7 +76,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { } } - var cpy [nblobs]restic.PackedBlob + var cpy [nblobs]*pack.PackedBlob b.ReportAllocs() b.ResetTimer() @@ -96,7 +97,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { idx := index.NewIndex() for i := 0; i < 5000; i++ { - idx.StorePack(restic.NewRandomID(), restic.Blobs{ + idx.StorePack(restic.NewRandomID(), pack.Blobs{ { BlobHandle: restic.NewRandomBlobHandle(), Length: 1234, @@ -133,7 +134,7 @@ func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (* } // buildPackfileWithoutHeader returns a manually built pack file without a header. -func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs restic.Blobs, packfile []byte) { +func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs pack.Blobs, packfile []byte) { opts := []zstd.EOption{ // Set the compression level configured. zstd.WithEncoderLevel(zstd.SpeedDefault), @@ -173,7 +174,7 @@ func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) ciphertextLength := after - before - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: id, @@ -280,19 +281,19 @@ func testStreamPack(t *testing.T, version uint) { // first, test regular usage t.Run("regular", func(t *testing.T) { tests := []struct { - blobs restic.Blobs + blobs pack.Blobs calls int shortFirstLoad bool }{ {packfileBlobs[1:2], 1, false}, {packfileBlobs[2:5], 1, false}, {packfileBlobs[2:8], 1, false}, - {restic.Blobs{ + {pack.Blobs{ packfileBlobs[0], packfileBlobs[4], packfileBlobs[2], }, 1, false}, - {restic.Blobs{ + {pack.Blobs{ packfileBlobs[0], packfileBlobs[len(packfileBlobs)-1], }, 2, false}, @@ -341,12 +342,12 @@ func testStreamPack(t *testing.T, version uint) { // next, test invalid uses, which should return an error t.Run("invalid", func(t *testing.T) { tests := []struct { - blobs restic.Blobs + blobs pack.Blobs err string }{ { // pass one blob several times - blobs: restic.Blobs{ + blobs: pack.Blobs{ packfileBlobs[3], packfileBlobs[8], packfileBlobs[3], @@ -357,7 +358,7 @@ func testStreamPack(t *testing.T, version uint) { { // pass something that's not a valid blob in the current pack file - blobs: restic.Blobs{ + blobs: pack.Blobs{ { Offset: 123, Length: 20000, @@ -368,7 +369,7 @@ func testStreamPack(t *testing.T, version uint) { { // pass a blob that's too small - blobs: restic.Blobs{ + blobs: pack.Blobs{ { Offset: 123, Length: 10, @@ -523,7 +524,7 @@ func TestStreamPackFallback(t *testing.T) { plaintext := rtest.Random(800, 42) blobID := restic.Hash(plaintext) - blobs := restic.Blobs{ + blobs := pack.Blobs{ { Length: uint(crypto.CiphertextLength(len(plaintext))), Offset: 0, diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 13320ddf7..083768a80 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -422,7 +422,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { rtest.OK(t, err) for pb := range idx.Values() { - packID := pb.PackID + packID := pb.PackID() if _, ok := packEntries[packID]; !ok { packEntries[packID] = make(map[restic.ID]struct{}) } diff --git a/internal/restic/blob.go b/internal/restic/blob.go index 8d1e8a720..5fad7b3f6 100644 --- a/internal/restic/blob.go +++ b/internal/restic/blob.go @@ -1,46 +1,11 @@ package restic import ( - "cmp" "fmt" - "slices" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" ) -// Blob is one part of a file or a tree. -type Blob struct { - BlobHandle - Length uint - Offset uint - UncompressedLength uint -} - -func (b Blob) String() string { - return fmt.Sprintf("", - b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength) -} - -func (b Blob) DataLength() uint { - if b.UncompressedLength != 0 { - return b.UncompressedLength - } - return uint(crypto.PlaintextLength(int(b.Length))) -} - -func (b Blob) IsCompressed() bool { - return b.UncompressedLength != 0 -} - -type Blobs []Blob - -func (b Blobs) Sort() { - slices.SortFunc(b, func(a, b Blob) int { - return cmp.Compare(a.Offset, b.Offset) - }) -} - // PackBlob is one index entry for a blob in a pack file. // The interface intentionally omits the offset at which a blob is stored in the pack. // This ensures that pack file internals are not leaked. @@ -54,29 +19,6 @@ type PackBlob interface { IsCompressed() bool } -// PackedBlob is a blob stored within a file. -type PackedBlob struct { - Blob - PackID ID -} - -type packBlob struct { - PackedBlob -} - -func (pb packBlob) PackID() ID { return pb.PackedBlob.PackID } - -func (pb packBlob) Handle() BlobHandle { return pb.BlobHandle } - -func (pb packBlob) CiphertextLength() uint { return pb.Length } - -func (pb packBlob) PlaintextLength() uint { return pb.DataLength() } - -func (pb packBlob) IsCompressed() bool { return pb.Blob.IsCompressed() } - -// AsPackBlob returns a PackBlob view of a PackedBlob. -func AsPackBlob(pb PackedBlob) PackBlob { return packBlob{pb} } - // BlobHandle identifies a blob of a given type. type BlobHandle struct { ID ID diff --git a/internal/restic/blob_test.go b/internal/restic/blob_test.go index 36e045792..951872250 100644 --- a/internal/restic/blob_test.go +++ b/internal/restic/blob_test.go @@ -3,8 +3,6 @@ package restic import ( "encoding/json" "testing" - - rtest "github.com/restic/restic/internal/test" ) var blobTypeJSON = []struct { @@ -41,20 +39,3 @@ func TestBlobTypeJSON(t *testing.T) { } } } - -func TestBlobsSort(t *testing.T) { - blobs := Blobs{ - {Offset: 100}, - {Offset: 0}, - {Offset: 50}, - } - blobs.Sort() - rtest.Equals(t, uint(0), blobs[0].Offset) - rtest.Equals(t, uint(50), blobs[1].Offset) - rtest.Equals(t, uint(100), blobs[2].Offset) -} - -func TestBlobsSortNilSlice(t *testing.T) { - var blobs Blobs - blobs.Sort() -} diff --git a/internal/restic/repository.go b/internal/restic/repository.go index db969a41a..b3e688271 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -124,11 +124,6 @@ type SaverRemoverUnpacked[FT FileTypes] interface { RemoverUnpacked[FT] } -type PackBlobs struct { - PackID ID - Blobs Blobs -} - type TerminalCounterFactory interface { // NewCounterTerminalOnly returns a new progress counter that is only shown if stdout points to a // terminal. It is not shown if --quiet or --json is specified.