From 064144284fc40e767bbf9c896146a0fd0165c181 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 Jun 2026 22:04:11 +0200 Subject: [PATCH] restic: change LookupBlob to return []PackBlob --- cmd/restic/cmd_copy.go | 6 +-- cmd/restic/cmd_find.go | 6 +-- cmd/restic/cmd_stats.go | 10 ++-- internal/checker/checker.go | 2 +- internal/repository/checker.go | 4 +- internal/repository/repack_test.go | 6 +-- internal/repository/repository.go | 9 +++- internal/repository/repository_test.go | 11 ++-- internal/restic/repository.go | 2 +- internal/restorer/filerestorer.go | 29 +++++----- internal/restorer/filerestorer_test.go | 74 ++++++++++++++++++-------- 11 files changed, 98 insertions(+), 61 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 908a61768..e2ca821ff 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -268,7 +268,7 @@ func copyTree(ctx context.Context, srcRepo *repository.Repository, dstRepo resti pb := srcRepo.LookupBlob(h.Type, h.ID) copyBlobs.Insert(h) for _, p := range pb { - packList.Insert(p.PackID) + packList.Insert(p.PackID()) } } } @@ -317,9 +317,9 @@ func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, pa countBlobs := 0 sizeBlobs := uint64(0) for blob := range copyBlobs.Keys() { - for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) { + for _, pb := range srcRepo.LookupBlob(blob.Type, blob.ID) { countBlobs++ - sizeBlobs += uint64(blob.Length) + sizeBlobs += uint64(pb.CiphertextLength()) break } } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 5069cfe7c..8dce2965d 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -577,9 +577,9 @@ func (f *Finder) findObjectPack(id string, t restic.BlobType) { } for _, b := range blobs { - if b.ID.Equal(rid) { - f.printer.S("Object belongs to pack %s", b.PackID) - f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String()) + if b.Handle().ID.Equal(rid) { + f.printer.S("Object belongs to pack %s", b.PackID()) + f.printer.S(" ... Pack %s: %v", b.PackID().String(), b.Handle()) break } } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 2e0883a8a..63ffd6d2a 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -166,16 +166,16 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) } - stats.TotalSize += uint64(pbs[0].Length) + stats.TotalSize += uint64(pbs[0].CiphertextLength()) if repo.Config().Version >= 2 { - stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength()))) if pbs[0].IsCompressed() { - stats.TotalCompressedBlobsSize += uint64(pbs[0].Length) - stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + stats.TotalCompressedBlobsSize += uint64(pbs[0].CiphertextLength()) + stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength()))) } } stats.TotalBlobCount++ - statsProgress.update(0, 1, uint64(pbs[0].Length)) + statsProgress.update(0, 1, uint64(pbs[0].CiphertextLength())) } if stats.TotalCompressedBlobsSize > 0 { stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 2e9941dd6..1ccc57199 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -308,7 +308,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID // convert used blobs into their encompassing packfiles for bh := range c.blobRefs.M.Keys() { for _, pb := range c.repo.LookupBlob(bh.Type, bh.ID) { - filteredPacks[pb.PackID] = allPacks[pb.PackID] + filteredPacks[pb.PackID()] = allPacks[pb.PackID()] } } diff --git a/internal/repository/checker.go b/internal/repository/checker.go index 3a6986192..0e44ef80a 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -464,8 +464,8 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest for _, blob := range blobs { // Check if blob is contained in index and position is correct idxHas := false - for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) { - if pb.PackID == id && pb.Blob == blob { + for _, pb := range r.idx.Lookup(blob.BlobHandle) { + if pb.PackID.Equal(id) && pb.Blob == blob { idxHas = true break } diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 7f9b04d92..999cb00ea 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -141,7 +141,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe } for _, pb := range list { - packs.Insert(pb.PackID) + packs.Insert(pb.PackID()) } } @@ -221,8 +221,8 @@ func testRepack(t *testing.T, version uint) { pb := list[0] - if removePacks.Has(pb.PackID) { - t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID) + if removePacks.Has(pb.PackID()) { + t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID()) } } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 4ecdc9859..d28ce64c9 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -670,8 +670,13 @@ func (r *Repository) Connections() uint { return r.be.Properties().Connections } -func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackedBlob { - return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) +func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob { + entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) + out := make([]restic.PackBlob, len(entries)) + for i, pb := range entries { + out[i] = restic.AsPackBlob(pb) + } + return out } // LookupBlobSize returns the size of blob id. Also returns pending blobs. diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index eddc8bd30..a775f9f43 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -211,7 +211,7 @@ func TestLoadBlobBroken(t *testing.T) { data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil) rtest.OK(t, err) rtest.Assert(t, bytes.Equal(buf, data), "data mismatch") - pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID() rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached") } @@ -400,11 +400,12 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { rtest.OK(t, err) for pb := range idx.Values() { - if _, ok := packEntries[pb.PackID]; !ok { - packEntries[pb.PackID] = make(map[restic.ID]struct{}) + packID := pb.PackID + if _, ok := packEntries[packID]; !ok { + packEntries[packID] = make(map[restic.ID]struct{}) } - packEntries[pb.PackID][id] = struct{}{} + packEntries[packID][id] = struct{}{} } return nil }) @@ -445,7 +446,7 @@ func TestListPack(t *testing.T) { repo.UseCache(c, t.Logf) // Forcibly cache pack file - packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID() rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil })) // Get size to list pack diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 1fe567b01..db969a41a 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -24,7 +24,7 @@ type Repository interface { LoadIndex(ctx context.Context, p TerminalCounterFactory) error - LookupBlob(t BlobType, id ID) []PackedBlob + LookupBlob(t BlobType, id ID) []PackBlob LookupBlobSize(t BlobType, id ID) (size uint, exists bool) NewAssociatedBlobSet() AssociatedBlobSet diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 5207f4f8d..3ec312a2c 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -47,7 +47,7 @@ type startWarmupFn func(context.Context, restic.IDSet) (restic.WarmupJob, error) // fileRestorer restores set of files type fileRestorer struct { - idx func(restic.BlobType, restic.ID) []restic.PackedBlob + idx func(restic.BlobType, restic.ID) []restic.PackBlob blobsLoader blobsLoaderFn startWarmup startWarmupFn @@ -68,7 +68,7 @@ type fileRestorer struct { func newFileRestorer(dst string, blobsLoader blobsLoaderFn, - idx func(restic.BlobType, restic.ID) []restic.PackedBlob, + idx func(restic.BlobType, restic.ID) []restic.PackBlob, connections uint, sparse bool, allowRecursiveDelete bool, @@ -102,7 +102,7 @@ func (r *fileRestorer) targetPath(location string) string { return filepath.Join(r.dst, location) } -func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int, fileOffset int64)) error { +func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(blob restic.PackBlob, idx int, fileOffset int64)) error { if len(blobIDs) == 0 { return nil } @@ -114,8 +114,8 @@ func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID return errors.Errorf("Unknown blob %s", blobID.String()) } pb := packs[0] - fn(pb.PackID, pb.Blob, i, fileOffset) - fileOffset += int64(pb.DataLength()) + fn(pb, i, fileOffset) + fileOffset += int64(pb.PlaintextLength()) } return nil @@ -143,14 +143,15 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { file.blobs = packsMap } restoredBlobs := false - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { + err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) { + packID := blob.PackID() if !file.state.HasMatchingBlob(idx) { if largeFile { - packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset}) + packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.Handle().ID, offset: fileOffset}) } restoredBlobs = true } else { - r.reportBlobProgress(file, uint64(blob.DataLength())) + r.reportBlobProgress(file, uint64(blob.PlaintextLength())) // completely ignore blob return } @@ -164,7 +165,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { packOrder = append(packOrder, packID) } pack.files[file] = struct{}{} - if blob.ID.Equal(r.zeroChunk) { + if blob.Handle().ID.Equal(r.zeroChunk) { file.sparse = r.sparse } }) @@ -278,9 +279,9 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { blobInfo.files[file] = append(blobInfo.files[file], fileOffset) } if fileBlobs, ok := file.blobs.(restic.IDs); ok { - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { - if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) { - addBlob(blob.BlobHandle, fileOffset) + err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) { + if blob.PackID().Equal(pack.id) && !file.state.HasMatchingBlob(idx) { + addBlob(blob.Handle(), fileOffset) } }) if err != nil { @@ -291,8 +292,8 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { for _, blob := range packsMap[pack.id] { idxPacks := r.idx(restic.DataBlob, blob.id) for _, idxPack := range idxPacks { - if idxPack.PackID.Equal(pack.id) { - addBlob(idxPack.BlobHandle, blob.offset) + if idxPack.PackID().Equal(pack.id) { + addBlob(idxPack.Handle(), blob.offset) break } } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index f5220998d..d517a741d 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -30,11 +30,32 @@ type TestWarmupJob struct { waitCalled bool } +type testPackBlob struct { + packID restic.ID + handle restic.BlobHandle + offset uint + ciphertext uint + plaintext uint + compressed bool +} + +var _ restic.PackBlob = (*testPackBlob)(nil) + +func (pb *testPackBlob) PackID() restic.ID { return pb.packID } + +func (pb *testPackBlob) Handle() restic.BlobHandle { return pb.handle } + +func (pb *testPackBlob) CiphertextLength() uint { return pb.ciphertext } + +func (pb *testPackBlob) PlaintextLength() uint { return pb.plaintext } + +func (pb *testPackBlob) IsCompressed() bool { return pb.compressed } + type TestRepo struct { packsIDToData map[restic.ID][]byte // blobs and files - blobs map[restic.ID][]restic.PackedBlob + blobs map[restic.ID][]restic.PackBlob files []*fileInfo filesPathToContent map[string]string @@ -44,7 +65,7 @@ type TestRepo struct { loader blobsLoaderFn } -func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackedBlob { +func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackBlob { packs := i.blobs[id] return packs } @@ -69,10 +90,16 @@ func (job *TestWarmupJob) Wait(_ context.Context) error { } func newTestRepo(content []TestFile) *TestRepo { + type packBlobLayout struct { + offset uint + ciphertext uint + plaintext uint + compressed bool + } type Pack struct { name string data []byte - blobs map[restic.ID]restic.Blob + blobs map[restic.ID]packBlobLayout } packs := make(map[string]Pack) filesPathToContent := make(map[string]string) @@ -86,21 +113,19 @@ func newTestRepo(content []TestFile) *TestRepo { var pack Pack var found bool if pack, found = packs[blob.pack]; !found { - pack = Pack{name: blob.pack, blobs: make(map[restic.ID]restic.Blob)} + pack = Pack{name: blob.pack, blobs: make(map[restic.ID]packBlobLayout)} } // calculate blob id and add to the pack as necessary blobID := restic.Hash([]byte(blob.data)) if _, found := pack.blobs[blobID]; !found { blobData := []byte(blob.data) - pack.blobs[blobID] = restic.Blob{ - BlobHandle: restic.BlobHandle{ - Type: restic.DataBlob, - ID: blobID, - }, - Length: uint(len(blobData)), - UncompressedLength: uint(len(blobData)), - Offset: uint(len(pack.data)), + n := uint(len(blobData)) + pack.blobs[blobID] = packBlobLayout{ + offset: uint(len(pack.data)), + ciphertext: n, + plaintext: n, + compressed: true, } pack.data = append(pack.data, blobData...) } @@ -110,14 +135,19 @@ func newTestRepo(content []TestFile) *TestRepo { filesPathToContent[file.name] = content } - blobs := make(map[restic.ID][]restic.PackedBlob) + blobs := make(map[restic.ID][]restic.PackBlob) packsIDToData := make(map[restic.ID][]byte) for _, pack := range packs { packID := restic.Hash(pack.data) packsIDToData[packID] = pack.data - for blobID, blob := range pack.blobs { - blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID}) + for blobID, layout := range pack.blobs { + blobs[blobID] = append(blobs[blobID], &testPackBlob{ + packID: packID, + handle: restic.BlobHandle{Type: restic.DataBlob, ID: blobID}, + offset: layout.offset, ciphertext: layout.ciphertext, + plaintext: layout.plaintext, compressed: layout.compressed, + }) } } @@ -138,12 +168,12 @@ func newTestRepo(content []TestFile) *TestRepo { warmupJobs: []*TestWarmupJob{}, } repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { - entries := make([]restic.PackedBlob, 0, len(handles)) + entries := make([]*testPackBlob, 0, len(handles)) for _, h := range handles { found := false for _, e := range repo.blobs[h.ID] { - if packID == e.PackID { - entries = append(entries, e) + if packID == e.PackID() { + entries = append(entries, e.(*testPackBlob)) found = true break } @@ -152,13 +182,13 @@ func newTestRepo(content []TestFile) *TestRepo { return fmt.Errorf("missing blob: %v", h) } } - slices.SortFunc(entries, func(a, b restic.PackedBlob) int { - return cmp.Compare(a.Offset, b.Offset) + slices.SortFunc(entries, func(a, b *testPackBlob) int { + return cmp.Compare(a.offset, b.offset) }) for _, e := range entries { - buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] - err := handleBlobFn(e.BlobHandle, buf, nil) + buf := repo.packsIDToData[packID][e.offset : e.offset+e.ciphertext] + err := handleBlobFn(e.handle, buf, nil) if err != nil { return err }