diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 7728af1a2..bf6e91f41 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -189,7 +189,7 @@ func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool { // copyTreeBatched copies multiple snapshots in one go. Snapshots are written after // data equivalent to at least 10 packfiles was written. -func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, +func copyTreeBatched(ctx context.Context, srcRepo *repository.Repository, dstRepo restic.Repository, selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error { // remember already processed trees across all snapshots @@ -254,7 +254,7 @@ func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo res return nil } -func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, +func copyTree(ctx context.Context, srcRepo *repository.Repository, dstRepo restic.Repository, visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) { copyBlobs := srcRepo.NewAssociatedBlobSet() @@ -268,7 +268,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep pb := srcRepo.LookupBlob(h.Type, h.ID) copyBlobs.Insert(h) for _, p := range pb { - packList.Insert(p.PackID) + packList.Insert(p.PackID()) } } } @@ -317,9 +317,9 @@ func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, pa countBlobs := 0 sizeBlobs := uint64(0) for blob := range copyBlobs.Keys() { - for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) { + for _, pb := range srcRepo.LookupBlob(blob.Type, blob.ID) { countBlobs++ - sizeBlobs += uint64(blob.Length) + sizeBlobs += uint64(pb.CiphertextLength()) break } } diff --git a/cmd/restic/cmd_copy_integration_test.go b/cmd/restic/cmd_copy_integration_test.go index 29e269578..55c7dbb7e 100644 --- a/cmd/restic/cmd_copy_integration_test.go +++ b/cmd/restic/cmd_copy_integration_test.go @@ -103,17 +103,17 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i defer unlock() rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - blobs, err := repo.ListPack(context.TODO(), id, size) + handles, err := repo.ListPackHandles(context.TODO(), id, size) rtest.OK(t, err) - rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob") + rtest.Assert(t, len(handles) > 0, "a packfile should contain at least one blob") - switch blobs[0].Type { + switch handles[0].Type { case restic.TreeBlob: countTreePacks++ case restic.DataBlob: countDataPacks++ } - countBlobs += len(blobs) + countBlobs += len(handles) return nil })) return nil diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index baa47dc1f..d669fe333 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -473,18 +473,18 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { delete(packIDs, idStr) } debug.Log("Found pack %s", idStr) - blobs, err := f.repo.ListPack(ctx, id, size) + handles, err := f.repo.ListPackHandles(ctx, id, size) if err != nil { return err } - for _, b := range blobs { - switch b.Type { + for _, h := range handles { + switch h.Type { case restic.DataBlob: - f.blobIDs[b.ID.String()] = struct{}{} + f.blobIDs[h.ID.String()] = struct{}{} case restic.TreeBlob: - f.treeIDs[b.ID.String()] = struct{}{} + f.treeIDs[h.ID.String()] = struct{}{} default: - panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String())) + panic(fmt.Sprintf("unknown type %v in blob list", h.Type.String())) } } // Stop searching when all packs have been found @@ -526,22 +526,23 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc // remember which packs were found in the index indexPackIDs := make(map[string]struct{}) - err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) { - idStr := pb.PackID.String() + err := f.repo.ListBlobs(wctx, func(pb restic.PackBlob) { + packID := pb.PackID() + idStr := packID.String() // keep entry in packIDs as Each() returns individual index entries matchingID := false if _, ok := packIDs[idStr]; ok { matchingID = true } else { - if _, ok := packIDs[pb.PackID.Str()]; ok { + if _, ok := packIDs[packID.Str()]; ok { // expand id - delete(packIDs, pb.PackID.Str()) + delete(packIDs, packID.Str()) packIDs[idStr] = struct{}{} matchingID = true } } if matchingID { - f.blobIDs[pb.ID.String()] = struct{}{} + f.blobIDs[pb.Handle().ID.String()] = struct{}{} indexPackIDs[idStr] = struct{}{} } }) @@ -577,9 +578,9 @@ func (f *Finder) findObjectPack(id string, t restic.BlobType) { } for _, b := range blobs { - if b.ID.Equal(rid) { - f.printer.S("Object belongs to pack %s", b.PackID) - f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String()) + if b.Handle().ID.Equal(rid) { + f.printer.S("Object belongs to pack %s", b.PackID()) + f.printer.S(" ... Pack %s: %v", b.PackID().String(), b.Handle()) break } } diff --git a/cmd/restic/cmd_find_integration_test.go b/cmd/restic/cmd_find_integration_test.go index e5e35348f..2587c2e01 100644 --- a/cmd/restic/cmd_find_integration_test.go +++ b/cmd/restic/cmd_find_integration_test.go @@ -179,9 +179,10 @@ func TestFindPackfile(t *testing.T) { packID := restic.ID{} done := false - err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) { - if !done && pb.Type == restic.TreeBlob { - packID = pb.PackID + err = repo.ListBlobs(ctx, func(pb restic.PackBlob) { + h := pb.Handle() + if !done && h.Type == restic.TreeBlob { + packID = pb.PackID() done = true } }) @@ -236,12 +237,12 @@ func TestFindPackID(t *testing.T) { // load Index rtest.OK(t, repo.LoadIndex(ctx, nil)) // go through all index entries and collect data and tree packfile(s) - rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - switch blob.Type { + rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) { + switch blob.Handle().Type { case restic.DataBlob: - dataPackID = blob.PackID + dataPackID = blob.PackID() case restic.TreeBlob: - treePackID = blob.PackID + treePackID = blob.PackID() } })) return nil diff --git a/cmd/restic/cmd_list_integration_test.go b/cmd/restic/cmd_list_integration_test.go index 6b2358d47..9e5cf666c 100644 --- a/cmd/restic/cmd_list_integration_test.go +++ b/cmd/restic/cmd_list_integration_test.go @@ -71,8 +71,8 @@ func testListBlobs(t testing.TB, gopts global.Options) (blobSetFromIndex restic. // get blobs from index blobSetFromIndex = restic.NewIDSet() - rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - blobSetFromIndex.Insert(blob.ID) + rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) { + blobSetFromIndex.Insert(blob.Handle().ID) })) return nil }) diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 72e8f94f8..e716d80e4 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -75,9 +75,10 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err // tree. If it is not referenced, we have a root tree. trees := make(map[restic.ID]bool) - err = repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - if blob.Type == restic.TreeBlob { - trees[blob.Blob.ID] = false + err = repo.ListBlobs(ctx, func(blob restic.PackBlob) { + h := blob.Handle() + if h.Type == restic.TreeBlob { + trees[h.ID] = false } }) if err != nil { diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 83d2a2954..f0b9fbd75 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -160,16 +160,16 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) } - stats.TotalSize += uint64(pbs[0].Length) + stats.TotalSize += uint64(pbs[0].CiphertextLength()) if repo.Config().Version >= 2 { - stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength()))) if pbs[0].IsCompressed() { - stats.TotalCompressedBlobsSize += uint64(pbs[0].Length) - stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength()))) + stats.TotalCompressedBlobsSize += uint64(pbs[0].CiphertextLength()) + stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength()))) } } stats.TotalBlobCount++ - statsProgress.Update(0, 1, uint64(pbs[0].Length)) + statsProgress.Update(0, 1, uint64(pbs[0].CiphertextLength())) } if stats.TotalCompressedBlobsSize > 0 { stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize) @@ -414,8 +414,8 @@ func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBl hist[i] = newSizeHistogram(2 * chunker.MaxSize) } - err := repo.ListBlobs(ctx, func(pb restic.PackedBlob) { - hist[pb.Type].Add(uint64(pb.Length)) + err := repo.ListBlobs(ctx, func(pb restic.PackBlob) { + hist[pb.Handle().Type].Add(uint64(pb.CiphertextLength())) }) return hist, err diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index d64447f7d..9f7b64259 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -269,9 +269,9 @@ func listTreePacks(gopts global.Options, t *testing.T) restic.IDSet { rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks = restic.NewIDSet() - return r.ListBlobs(ctx, func(pb restic.PackedBlob) { - if pb.Type == restic.TreeBlob { - treePacks.Insert(pb.PackID) + return r.ListBlobs(ctx, func(pb restic.PackBlob) { + if pb.Handle().Type == restic.TreeBlob { + treePacks.Insert(pb.PackID()) } }) }) @@ -318,9 +318,9 @@ func removePacksExcept(gopts global.Options, t testing.TB, keep restic.IDSet, re rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks := restic.NewIDSet() - rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) { - if pb.Type == restic.TreeBlob { - treePacks.Insert(pb.PackID) + rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackBlob) { + if pb.Handle().Type == restic.TreeBlob { + treePacks.Insert(pb.PackID()) } })) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 522729a16..1ccc57199 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -277,8 +277,8 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er ctx, cancel := context.WithCancel(ctx) defer cancel() - err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} + err = c.repo.ListBlobs(ctx, func(blob restic.PackBlob) { + h := blob.Handle() if !c.blobRefs.M.Has(h) { debug.Log("blob %v not referenced", h) blobs = append(blobs, h) @@ -308,7 +308,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID // convert used blobs into their encompassing packfiles for bh := range c.blobRefs.M.Keys() { for _, pb := range c.repo.LookupBlob(bh.Type, bh.ID) { - filteredPacks[pb.PackID] = allPacks[pb.PackID] + filteredPacks[pb.PackID()] = allPacks[pb.PackID()] } } diff --git a/internal/repository/checker.go b/internal/repository/checker.go index c3f530abf..351dbc247 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -87,16 +87,18 @@ func NewChecker(repo *Repository) *Checker { } func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) { packs := make(map[restic.ID]restic.BlobType) - err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) { - tpe, exists := packs[pb.PackID] + err := idx.ListBlobs(ctx, func(pb restic.PackBlob) { + packID := pb.PackID() + h := pb.Handle() + tpe, exists := packs[packID] if exists { - if pb.Type != tpe { + if h.Type != tpe { tpe = restic.InvalidBlob } } else { - tpe = pb.Type + tpe = h.Type } - packs[pb.PackID] = tpe + packs[packID] = tpe }) return packs, err } @@ -127,10 +129,11 @@ func (c *Checker) LoadIndex(ctx context.Context, p restic.TerminalCounterFactory } cnt++ - if _, ok := packToIndex[blob.PackID]; !ok { - packToIndex[blob.PackID] = restic.NewIDSet() + packID := blob.PackID() + if _, ok := packToIndex[packID]; !ok { + packToIndex[packID] = restic.NewIDSet() } - packToIndex[blob.PackID].Insert(id) + packToIndex[packID].Insert(id) } for pbs := range idx.EachByPack(ctx, restic.NewIDSet()) { @@ -257,7 +260,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID type checkTask struct { id restic.ID size int64 - blobs restic.Blobs + blobs pack.Blobs } ch := make(chan checkTask) @@ -306,7 +309,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID } // push packs to ch - for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) { + for pbs := range c.repo.listPacksFromIndex(ctx, packSet) { size := packs[pbs.PackID] debug.Log("listed %v", pbs.PackID) select { @@ -327,7 +330,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID } // checkPack reads a pack and checks the integrity of all blobs. -func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { +func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec) if err != nil { if r.cache != nil { @@ -346,7 +349,7 @@ func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Bl return err } -func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { +func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { type partialReadError struct { error @@ -462,8 +465,8 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest for _, blob := range blobs { // Check if blob is contained in index and position is correct idxHas := false - for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) { - if pb.PackID == id && pb.Blob == blob { + for _, pb := range r.idx.Lookup(blob.BlobHandle) { + if pb.PackID().Equal(id) && pb.Blob == blob { idxHas = true break } diff --git a/internal/repository/checker_test.go b/internal/repository/checker_test.go index def9a9951..c1a71408c 100644 --- a/internal/repository/checker_test.go +++ b/internal/repository/checker_test.go @@ -21,7 +21,7 @@ import ( var checkerTestData = filepath.Join("..", "checker", "testdata", "checker-test-repo.tar.gz") func testWrapCheckPack(ctx context.Context, t *testing.T, repo *Repository, - packID restic.ID, blobs []restic.Blob, size int64, + packID restic.ID, blobs pack.Blobs, size int64, ) error { t.Helper() bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize) @@ -46,8 +46,8 @@ func TestGapInBlobs(t *testing.T) { _, ok := repoPacks[packID] rtest.Assert(t, ok, "expected pack 19a731a515618ec8b75fc0ff3b887d8feb83aef1001c9899f6702761142ed068") - blobs := []restic.Blob{} - pb := <-repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(packID)) + blobs := pack.Blobs{} + pb := <-repo.listPacksFromIndex(context.TODO(), restic.NewIDSet(packID)) blobs = append(blobs, pb.Blobs...) // assertion for clarity, actually can't fail as the packfile content is fixed diff --git a/internal/repository/debug.go b/internal/repository/debug.go index c6691380c..dc0a81ce2 100644 --- a/internal/repository/debug.go +++ b/internal/repository/debug.go @@ -49,7 +49,7 @@ func writePackDumpJSON(wr io.Writer, item any) error { func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error { var m sync.Mutex return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { - blobs, err := repo.ListPack(ctx, id, size) + blobs, err := repo.listPack(ctx, id, size) if err != nil { printer.E("error for pack %v: %v", id.Str(), err) return nil @@ -116,15 +116,14 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami blobsLoaded := false // examine all data the indexes have for the pack file - for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { - blobs := b.Blobs - if len(blobs) == 0 { + for b := range repo.listPacksFromIndex(ctx, restic.NewIDSet(id)) { + if len(b.Blobs) == 0 { continue } - checkPackSize(blobs, len(buf), printer) + checkPackSize(b.Blobs, len(buf), printer) - err = loadBlobs(ctx, opts, repo, id, blobs, printer) + err := loadBlobs(ctx, opts, repo, id, b.Blobs, printer) if err != nil { printer.E("error: %v", err) } else { @@ -135,7 +134,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami printer.S(" ========================================") printer.S(" inspect the pack itself") - blobs, err := repo.ListPack(ctx, id, int64(len(buf))) + blobs, err := repo.listPack(ctx, id, int64(len(buf))) if err != nil { return fmt.Errorf("pack %v: %v", id.Str(), err) } @@ -147,7 +146,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami return nil } -func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { +func checkPackSize(blobs pack.Blobs, fileSize int, printer progress.Printer) { // track current size and offset var size, offset uint64 @@ -285,7 +284,7 @@ func decryptUnsigned(k *crypto.Key, buf []byte) []byte { return out } -func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { +func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list pack.Blobs, printer progress.Printer) error { dec, err := zstd.NewReader(nil) if err != nil { panic(err) diff --git a/internal/repository/index/associated_data.go b/internal/repository/index/associated_data.go index 1267bdc4f..cd66f5c4e 100644 --- a/internal/repository/index/associated_data.go +++ b/internal/repository/index/associated_data.go @@ -159,14 +159,15 @@ func (a *AssociatedSet[T]) All() iter.Seq2[restic.BlobHandle, T] { } for pb := range a.idx.Values() { - if _, ok := a.overflow[pb.BlobHandle]; ok { + bh := pb.Handle() + if _, ok := a.overflow[bh]; ok { // already reported via overflow set continue } - val, known := a.Get(pb.BlobHandle) + val, known := a.Get(bh) if known { - if !yield(pb.BlobHandle, val) { + if !yield(bh, val) { return } } diff --git a/internal/repository/index/associated_data_test.go b/internal/repository/index/associated_data_test.go index 07e0a3d58..413f60990 100644 --- a/internal/repository/index/associated_data_test.go +++ b/internal/repository/index/associated_data_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -19,17 +20,17 @@ func (n *noopSaver) SaveUnpacked(_ context.Context, _ restic.FileType, buf []byt return restic.Hash(buf), nil } -func makeFakePackedBlob() (restic.BlobHandle, restic.PackedBlob) { +func makeFakePackedBlob() (restic.BlobHandle, *pack.PackedBlob) { bh := restic.NewRandomBlobHandle() - blob := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bh, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - return bh, blob + return bh, pb } func list(bs *AssociatedSet[uint8]) restic.BlobHandles { @@ -40,7 +41,7 @@ func TestAssociatedSet(t *testing.T) { bh, blob := makeFakePackedBlob() mi := NewMasterIndex() - test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) @@ -123,14 +124,14 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) { _, blob := makeFakePackedBlob() mi := NewMasterIndex() - test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) // add new blobs to index after building the set of, blob2 := makeFakePackedBlob() - test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) // non-existent @@ -167,10 +168,10 @@ func TestAssociatedSetIntersectAndSub(t *testing.T) { bh3, blob3 := makeFakePackedBlob() bh4, blob4 := makeFakePackedBlob() - test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, restic.Blobs{blob3.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, restic.Blobs{blob4.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob3.PackID(), pack.Blobs{blob3.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob4.PackID(), pack.Blobs{blob4.Blob}, saver)) test.OK(t, mi.Flush(context.TODO(), saver)) t.Run("Intersect", func(t *testing.T) { diff --git a/internal/repository/index/index.go b/internal/repository/index/index.go index abd8eda54..9d205ba4b 100644 --- a/internal/repository/index/index.go +++ b/internal/repository/index/index.go @@ -73,7 +73,7 @@ func (idx *Index) addToPacks(id restic.ID) int { return len(idx.packs) - 1 } -func (idx *Index) store(packIndex int, blob restic.Blob) { +func (idx *Index) store(packIndex int, blob pack.Blob) { // assert that offset and length fit into uint32! if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 { panic("offset or length does not fit in uint32. You have packs > 4GB!") @@ -146,7 +146,7 @@ func (idx *Index) Preallocate(t restic.BlobType, numEntries int) { // StorePack remembers the ids of all blobs of a given pack // in the index -func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) { +func (idx *Index) StorePack(id restic.ID, blobs pack.Blobs) { idx.m.Lock() defer idx.m.Unlock() @@ -162,23 +162,24 @@ func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) { } } -func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) restic.PackedBlob { - return restic.PackedBlob{ - Blob: restic.Blob{ +func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) *pack.PackedBlob { + return &pack.PackedBlob{ + Pack: idx.packs[e.packIndex], + Blob: pack.Blob{ BlobHandle: restic.BlobHandle{ ID: e.id, - Type: t}, + Type: t, + }, Length: uint(e.length), Offset: uint(e.offset), UncompressedLength: uint(e.uncompressedLength), }, - PackID: idx.packs[e.packIndex], } } // Lookup queries the index for the blob ID and returns all entries including -// duplicates. Adds found entries to blobs and returns the result. -func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic.PackedBlob { +// duplicates. Adds found entries to pbs and returns the result. +func (idx *Index) Lookup(bh restic.BlobHandle, pbs []*pack.PackedBlob) []*pack.PackedBlob { idx.m.RLock() defer idx.m.RUnlock() @@ -215,8 +216,8 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found // Values returns an iterator over all blobs known to the index. This blocks any // modification of the index. -func (idx *Index) Values() iter.Seq[restic.PackedBlob] { - return func(yield func(restic.PackedBlob) bool) { +func (idx *Index) Values() iter.Seq[*pack.PackedBlob] { + return func(yield func(*pack.PackedBlob) bool) { idx.m.RLock() defer idx.m.RUnlock() @@ -231,17 +232,23 @@ func (idx *Index) Values() iter.Seq[restic.PackedBlob] { } } -// EachByPack returns a channel that yields all blobs known to the index +// PackBlobs lists all blobs contained in a pack file according to the index. +type PackBlobs struct { + PackID restic.ID + Blobs pack.Blobs +} + +// EachByPack returns a channel that yields all blobs known to the index, // grouped by packID but ignoring blobs with a packID in packPlacklist for // finalized indexes. // This filtering is used when rebuilding the index where we need to ignore packs // from the finalized index which have been re-read into a non-finalized index. // When the context is cancelled, the background goroutine // terminates. This blocks any modification of the index. -func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan restic.PackBlobs { +func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan PackBlobs { idx.m.RLock() - ch := make(chan restic.PackBlobs) + ch := make(chan PackBlobs) go func() { defer idx.m.RUnlock() @@ -262,7 +269,7 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <- } for packID, packByType := range byPack { - var result restic.PackBlobs + var result PackBlobs result.PackID = packID for typ, p := range packByType { for _, e := range p { @@ -482,7 +489,7 @@ func (idx *Index) merge(idx2 *Index) error { for e := range m.valuesWithID(e2.id) { b := idx.toPackedBlob(e, restic.BlobType(typ)) b2 := idx2.toPackedBlob(e2, restic.BlobType(typ)) - if b == b2 { + if *b == *b2 { found = true break } @@ -519,7 +526,7 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, err error) { packID := idx.addToPacks(p.ID) for _, blob := range p.Blobs { - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: blob.Type, ID: blob.ID}, @@ -550,7 +557,7 @@ func (idx *Index) Len(t restic.BlobType) uint { return idx.byType[t].len() } -func PackBlobsHash(pbs restic.PackBlobs) restic.ID { +func PackBlobsHash(pbs PackBlobs) restic.ID { h := sha256.New() h.Write(pbs.PackID[:]) diff --git a/internal/repository/index/index_internal_test.go b/internal/repository/index/index_internal_test.go index 67591ef8a..94a871ed3 100644 --- a/internal/repository/index/index_internal_test.go +++ b/internal/repository/index/index_internal_test.go @@ -14,7 +14,7 @@ func TestIndexOversized(t *testing.T) { // Add blobs up to indexMaxBlobs + pack.MaxHeaderEntries - 1 packID := idx.addToPacks(restic.NewRandomID()) for i := uint(0); i < indexMaxBlobs+pack.MaxHeaderEntries-1; i++ { - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), @@ -27,7 +27,7 @@ func TestIndexOversized(t *testing.T) { rtest.Assert(t, !Oversized(idx), "index should not be considered oversized") // Add one more blob to exceed the limit - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), diff --git a/internal/repository/index/index_test.go b/internal/repository/index/index_test.go index 2a53072df..d5a779da3 100644 --- a/internal/repository/index/index_test.go +++ b/internal/repository/index/index_test.go @@ -9,19 +9,20 @@ import ( "testing" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func TestIndexSerialize(t *testing.T) { - tests := []restic.PackedBlob{} + tests := []*pack.PackedBlob{} idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 20; j++ { @@ -31,14 +32,14 @@ func TestIndexSerialize(t *testing.T) { // test a mix of compressed and uncompressed packs uncompressedLength = 2 * length } - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, UncompressedLength: uncompressedLength, }, - PackID: packID, } blobs = append(blobs, pb.Blob) tests = append(tests, pb) @@ -64,17 +65,17 @@ func TestIndexSerialize(t *testing.T) { rtest.OK(t, err) for _, testBlob := range tests { - list := idx.Lookup(testBlob.BlobHandle, nil) + list := idx.Lookup(testBlob.Handle(), nil) if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list) } result := list[0] rtest.Equals(t, testBlob, result) - list2 := idx2.Lookup(testBlob.BlobHandle, nil) + list2 := idx2.Lookup(testBlob.Handle(), nil) if len(list2) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list2), list2) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list2), list2) } result2 := list2[0] @@ -82,21 +83,21 @@ func TestIndexSerialize(t *testing.T) { } // add more blobs to idx - newtests := []restic.PackedBlob{} + newtests := []*pack.PackedBlob{} for i := 0; i < 10; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 10; j++ { length := uint(i*100 + j) - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, }, - PackID: packID, } blobs = append(blobs, pb.Blob) newtests = append(newtests, pb) @@ -127,9 +128,9 @@ func TestIndexSerialize(t *testing.T) { // all new blobs must be in the index for _, testBlob := range newtests { - list := idx3.Lookup(testBlob.BlobHandle, nil) + list := idx3.Lookup(testBlob.Handle(), nil) if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list) } blob := list[0] @@ -145,12 +146,12 @@ func TestIndexSize(t *testing.T) { blobCount := 100 for i := 0; i < packs; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < blobCount; j++ { length := uint(i*100 + j) - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, @@ -293,15 +294,15 @@ func TestIndexUnserialize(t *testing.T) { t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) - rtest.Equals(t, test.packID, blob.PackID) - rtest.Equals(t, test.tpe, blob.Type) - rtest.Equals(t, test.offset, blob.Offset) - rtest.Equals(t, test.length, blob.Length) + rtest.Equals(t, test.packID, blob.PackID()) + rtest.Equals(t, test.tpe, blob.Blob.Type) + rtest.Equals(t, test.offset, blob.Blob.Offset) + rtest.Equals(t, test.length, blob.Blob.Length) switch task.version { case 1: - rtest.Equals(t, uint(0), blob.UncompressedLength) + rtest.Equals(t, uint(0), blob.Blob.UncompressedLength) case 2: - rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength) + rtest.Equals(t, test.uncompressedLength, blob.Blob.UncompressedLength) default: t.Fatal("Invalid index version") } @@ -313,20 +314,20 @@ func TestIndexUnserialize(t *testing.T) { } for _, blob := range blobs { - b, ok := exampleLookupTest.blobs[blob.ID] + b, ok := exampleLookupTest.blobs[blob.Handle().ID] if !ok { - t.Errorf("unexpected blob %v found", blob.ID.Str()) + t.Errorf("unexpected blob %v found", blob.Handle().ID.String()) } - if blob.Type != b { - t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) + if blob.Blob.Type != b { + t.Errorf("unexpected type for blob %v: want %v, got %v", blob.Handle().ID.String(), b, blob.Blob.Type) } } } } -func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) { +func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []*pack.PackedBlob) { for pb := range idx.Values() { - if pb.PackID.Equal(id) { + if pb.PackID().Equal(id) { pbs = append(pbs, pb) } } @@ -401,7 +402,7 @@ func TestIndexPacks(t *testing.T) { for i := 0; i < 20; i++ { packID := restic.NewRandomID() - idx.StorePack(packID, restic.Blobs{ + idx.StorePack(packID, pack.Blobs{ { BlobHandle: restic.NewRandomBlobHandle(), Offset: 0, @@ -433,12 +434,12 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupB // create index with given number of pack files for i := 0; i < packfiles; i++ { packID := NewRandomTestID(rng) - var blobs restic.Blobs + var blobs pack.Blobs offset := 0 for offset < maxPackSize { size := 2000 + rng.Intn(4*1024*1024) id := NewRandomTestID(rng) - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: id, @@ -482,7 +483,7 @@ func BenchmarkIndexHasKnown(b *testing.B) { idx, _ := createRandomIndex(rand.New(rand.NewSource(0)), 200000) handles := make([]restic.BlobHandle, 0, 100000) for handle := range idx.Values() { - handles = append(handles, handle.BlobHandle) + handles = append(handles, handle.Handle()) if len(handles) == cap(handles) { break } @@ -517,14 +518,14 @@ func BenchmarkIndexAllocParallel(b *testing.B) { } func TestIndexHas(t *testing.T) { - tests := []restic.PackedBlob{} + tests := []*pack.PackedBlob{} idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 20; j++ { @@ -534,14 +535,14 @@ func TestIndexHas(t *testing.T) { // test a mix of compressed and uncompressed packs uncompressedLength = 2 * length } - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, UncompressedLength: uncompressedLength, }, - PackID: packID, } blobs = append(blobs, pb.Blob) tests = append(tests, pb) @@ -551,11 +552,11 @@ func TestIndexHas(t *testing.T) { } for _, testBlob := range tests { - rtest.Assert(t, idx.Has(testBlob.BlobHandle), "Index reports not having data blob added to it") + rtest.Assert(t, idx.Has(testBlob.Handle()), "Index reports not having data blob added to it") } rtest.Assert(t, !idx.Has(restic.NewRandomBlobHandle()), "Index reports having a data blob not added to it") - rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob") + rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].Handle().ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob") } func TestMixedEachByPack(t *testing.T) { @@ -566,7 +567,7 @@ func TestMixedEachByPack(t *testing.T) { for i := 0; i < 50; i++ { packID := restic.NewRandomID() expected[packID] = 1 - blobs := restic.Blobs{ + blobs := pack.Blobs{ { BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, Offset: 0, @@ -608,7 +609,7 @@ func TestEachByPackIgnoes(t *testing.T) { } else { expected[packID] = 1 } - blobs := restic.Blobs{ + blobs := pack.Blobs{ { BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, Offset: 0, diff --git a/internal/repository/index/master_index.go b/internal/repository/index/master_index.go index 84db0d962..a667e6f4f 100644 --- a/internal/repository/index/master_index.go +++ b/internal/repository/index/master_index.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" @@ -39,10 +40,11 @@ func (mi *MasterIndex) clearPendingBlobs() { } // Lookup queries all known Indexes for the ID and returns all matches. -func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) { +func (mi *MasterIndex) Lookup(bh restic.BlobHandle) []*pack.PackedBlob { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() + var pbs []*pack.PackedBlob for _, idx := range mi.idx { pbs = idx.Lookup(bh, pbs) } @@ -145,12 +147,12 @@ func (mi *MasterIndex) Insert(idx *Index) { } // StorePack remembers the id and pack in the index. -func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs restic.Blobs, r restic.SaverUnpacked[restic.FileType]) error { +func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs pack.Blobs, r restic.SaverUnpacked[restic.FileType]) error { mi.storePack(id, blobs) return mi.saveFullIndex(ctx, r) } -func (mi *MasterIndex) storePack(id restic.ID, blobs restic.Blobs) { +func (mi *MasterIndex) storePack(id restic.ID, blobs pack.Blobs) { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() @@ -218,8 +220,8 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index { // Values returns an iterator over all blobs known to the index. This blocks any // modification of the index. -func (mi *MasterIndex) Values() iter.Seq[restic.PackedBlob] { - return func(yield func(restic.PackedBlob) bool) { +func (mi *MasterIndex) Values() iter.Seq[*pack.PackedBlob] { + return func(yield func(*pack.PackedBlob) bool) { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() @@ -663,13 +665,13 @@ func (mi *MasterIndex) saveFullIndex(ctx context.Context, r restic.SaverUnpacked } // ListPacks returns the blobs of the specified pack files grouped by pack file. -func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { - out := make(chan restic.PackBlobs) +func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan PackBlobs { + out := make(chan PackBlobs) go func() { defer close(out) // only resort a part of the index to keep the memory overhead bounded for i := byte(0); i < 16; i++ { - packBlob := make(map[restic.ID]restic.Blobs) + packBlob := make(map[restic.ID]pack.Blobs) for pack := range packs { if pack[0]&0xf == i { packBlob[pack] = nil @@ -682,8 +684,9 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan if ctx.Err() != nil { return } - if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i { - packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob) + packID := pb.PackID() + if packs.Has(packID) && packID[0]&0xf == i { + packBlob[packID] = append(packBlob[packID], pb.Blob) } } @@ -692,7 +695,7 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan // allow GC packBlob[packID] = nil select { - case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}: + case out <- PackBlobs{PackID: packID, Blobs: pbs}: case <-ctx.Done(): return } diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index 56d01a503..ed320cc3b 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/progress" @@ -24,18 +25,18 @@ func TestMasterIndex(t *testing.T) { bhInIdx2 := restic.NewRandomBlobHandle() bhInIdx12 := restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob} - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx1, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx2, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -43,9 +44,9 @@ func TestMasterIndex(t *testing.T) { }, } - blob12a := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob12a := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx12, Length: uint(crypto.CiphertextLength(123)), Offset: 110, @@ -53,9 +54,9 @@ func TestMasterIndex(t *testing.T) { }, } - blob12b := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob12b := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx12, Length: uint(crypto.CiphertextLength(123)), Offset: 50, @@ -64,12 +65,12 @@ func TestMasterIndex(t *testing.T) { } idx1 := index.NewIndex() - idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) - idx1.StorePack(blob12a.PackID, restic.Blobs{blob12a.Blob}) + idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) + idx1.StorePack(blob12a.PackID(), pack.Blobs{blob12a.Blob}) idx2 := index.NewIndex() - idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) - idx2.StorePack(blob12b.PackID, restic.Blobs{blob12b.Blob}) + idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) + idx2.StorePack(blob12b.PackID(), pack.Blobs{blob12b.Blob}) mIdx := index.NewMasterIndex() mIdx.Insert(idx1) @@ -77,7 +78,7 @@ func TestMasterIndex(t *testing.T) { // test idInIdx1 blobs := mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) size, found := mIdx.LookupSize(bhInIdx1) rtest.Equals(t, true, found) @@ -85,7 +86,7 @@ func TestMasterIndex(t *testing.T) { // test idInIdx2 blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) size, found = mIdx.LookupSize(bhInIdx2) rtest.Equals(t, true, found) @@ -95,19 +96,20 @@ func TestMasterIndex(t *testing.T) { blobs = mIdx.Lookup(bhInIdx12) rtest.Equals(t, 2, len(blobs)) - // test Lookup result for blob12a - found = false - if blobs[0] == blob12a || blobs[1] == blob12a { - found = true + containsPackedBlob := func(list []*pack.PackedBlob, want *pack.PackedBlob) bool { + for _, b := range list { + if b.PackID().Equal(want.PackID()) && b.Blob == want.Blob { + return true + } + } + return false } - rtest.Assert(t, found, "blob12a not found in result") + + // test Lookup result for blob12a + rtest.Assert(t, containsPackedBlob(blobs, blob12a), "blob12a not found in result") // test Lookup result for blob12b - found = false - if blobs[0] == blob12b || blobs[1] == blob12b { - found = true - } - rtest.Assert(t, found, "blob12a not found in result") + rtest.Assert(t, containsPackedBlob(blobs, blob12b), "blob12b not found in result") size, found = mIdx.LookupSize(bhInIdx12) rtest.Equals(t, true, found) @@ -135,7 +137,7 @@ func TestMasterIndexAddPending(t *testing.T) { // Test AddPending: try to add a blob that's already in an index (should return false) bhInIndex := restic.NewRandomBlobHandle() idx := index.NewIndex() - idx.StorePack(restic.NewRandomID(), restic.Blobs{{ + idx.StorePack(restic.NewRandomID(), pack.Blobs{{ BlobHandle: bhInIndex, Length: uint(crypto.CiphertextLength(50)), Offset: 0, @@ -173,14 +175,14 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) { // Store the blob in a pack packID := restic.NewRandomID() - blob := restic.Blob{ + blob := pack.Blob{ BlobHandle: bhPending, Length: uint(crypto.CiphertextLength(75)), Offset: 0, UncompressedLength: 75, } saver := &noopSaver{} - err := mIdx.StorePack(context.Background(), packID, restic.Blobs{blob}, saver) + err := mIdx.StorePack(context.Background(), packID, pack.Blobs{blob}, saver) rtest.OK(t, err) // Verify it is still found @@ -191,8 +193,8 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) { // Verify the blob can be found via Lookup from the index blobs := mIdx.Lookup(bhPending) rtest.Assert(t, len(blobs) > 0, "blob should be found in index after StorePack") - rtest.Equals(t, packID, blobs[0].PackID) - rtest.Equals(t, bhPending, blobs[0].BlobHandle) + rtest.Equals(t, packID, blobs[0].PackID()) + rtest.Equals(t, bhPending, blobs[0].Handle()) // Test that adding the same blob as pending again fails (it's now in index) added = mIdx.AddPending(bhPending, 100) @@ -203,18 +205,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) { bhInIdx1 := restic.NewRandomBlobHandle() bhInIdx2 := restic.NewRandomBlobHandle() - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx1, Length: 10, Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx2, Length: 100, Offset: 10, @@ -223,10 +225,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) { } idx1 := index.NewIndex() - idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) + idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) idx2 := index.NewIndex() - idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) + idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) mIdx := index.NewMasterIndex() mIdx.Insert(idx1) @@ -246,18 +248,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) { rtest.Equals(t, 2, blobCount) blobs := mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) blobs = mIdx.Lookup(restic.NewRandomBlobHandle()) rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") // merge another index containing identical blobs idx3 := index.NewIndex() - idx3.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) - idx3.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) + idx3.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) + idx3.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) mIdx.Insert(idx3) finalIndexes, idxCount, newIDs := index.TestMergeIndex(t, mIdx) @@ -268,10 +270,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) { // Index should have same entries as before! blobs = mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) blobCount = 0 for range mIdx.Values() { @@ -448,9 +450,9 @@ func testIndexSave(t *testing.T, version uint) { idx := index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) - blobs := make(map[restic.PackedBlob]struct{}) + blobs := make(map[pack.PackedBlob]struct{}) for pb := range idx.Values() { - blobs[pb] = struct{}{} + blobs[*pb] = struct{}{} } rtest.OK(t, test.saver(idx, unpacked)) @@ -458,8 +460,8 @@ func testIndexSave(t *testing.T, version uint) { rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) for pb := range idx.Values() { - if _, ok := blobs[pb]; ok { - delete(blobs, pb) + if _, ok := blobs[*pb]; ok { + delete(blobs, *pb) } else { t.Fatalf("unexpected blobs %v", pb) } @@ -481,9 +483,9 @@ func testIndexSavePartial(t *testing.T, version uint) { // capture blob list before adding fourth snapshot idx := index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) - blobs := make(map[restic.PackedBlob]struct{}) + blobs := make(map[pack.PackedBlob]struct{}) for pb := range idx.Values() { - blobs[pb] = struct{}{} + blobs[*pb] = struct{}{} } // add+remove new snapshot and track its pack files @@ -502,8 +504,8 @@ func testIndexSavePartial(t *testing.T, version uint) { idx = index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) for pb := range idx.Values() { - if _, ok := blobs[pb]; ok { - delete(blobs, pb) + if _, ok := blobs[*pb]; ok { + delete(blobs, *pb) } else { t.Fatalf("unexpected blobs %v", pb) } @@ -516,7 +518,7 @@ func testIndexSavePartial(t *testing.T, version uint) { checker.TestCheckRepo(t, repo) } -func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[restic.PackedBlob]struct{} { +func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[pack.PackedBlob]struct{} { p := progress.NewCounter(0, 0, nil) rtest.OK(t, master.Load(context.TODO(), repo, p, nil)) v, max := p.Get() @@ -525,10 +527,10 @@ func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, ma return collectBlobs(master) } -func collectBlobs(master *index.MasterIndex) map[restic.PackedBlob]struct{} { - s := make(map[restic.PackedBlob]struct{}) +func collectBlobs(master *index.MasterIndex) map[pack.PackedBlob]struct{} { + s := make(map[pack.PackedBlob]struct{}) for pb := range master.Values() { - s[pb] = struct{}{} + s[*pb] = struct{}{} } return s } @@ -588,17 +590,17 @@ func TestRewriteOversizedIndex(t *testing.T) { return idx.Len(restic.DataBlob) > 2*fullIndexCount } - var blobs restic.Blobs + var blobs pack.Blobs // build oversized index idx := index.NewIndex() numPacks := 5 for p := 0; p < numPacks; p++ { packID := restic.NewRandomID() - packBlobs := make(restic.Blobs, 0, fullIndexCount) + packBlobs := make(pack.Blobs, 0, fullIndexCount) for i := 0; i < fullIndexCount; i++ { - blob := restic.Blob{ + blob := pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), @@ -645,17 +647,17 @@ func TestRewriteSplitPacks(t *testing.T) { bh2 := restic.NewRandomBlobHandle() bhOther := restic.NewRandomBlobHandle() - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bh1, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: blob1.PackID, - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: blob1.PackID(), + Blob: pack.Blob{ BlobHandle: bh2, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -663,9 +665,9 @@ func TestRewriteSplitPacks(t *testing.T) { }, } // used to force index repacking - blobOther := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blobOther := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhOther, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -673,25 +675,25 @@ func TestRewriteSplitPacks(t *testing.T) { } mi := index.NewMasterIndex() - rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID), nil, nil, index.MasterIndexRewriteOpts{})) + rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID()), nil, nil, index.MasterIndexRewriteOpts{})) mi = index.NewMasterIndex() rtest.OK(t, mi.Load(context.TODO(), repo, nil, nil)) // test that all blobs are still in the index - for _, blob := range []restic.PackedBlob{blob1, blob2} { - blobs := mi.Lookup(blob.BlobHandle) - rtest.Equals(t, []restic.PackedBlob{blob}, blobs) + for _, blob := range []*pack.PackedBlob{blob1, blob2} { + blobs := mi.Lookup(blob.Handle()) + rtest.Equals(t, []*pack.PackedBlob{blob}, blobs) } - blobs := mi.Lookup(blobOther.BlobHandle) + blobs := mi.Lookup(blobOther.Handle()) rtest.Equals(t, nil, blobs) } @@ -713,17 +715,17 @@ func TestRewriteFullPacks(t *testing.T) { packA := restic.NewRandomID() packB := restic.NewRandomID() - blobA := restic.PackedBlob{ - PackID: packA, - Blob: restic.Blob{ + blobA := &pack.PackedBlob{ + Pack: packA, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blobB := restic.PackedBlob{ - PackID: packB, - Blob: restic.Blob{ + blobB := &pack.PackedBlob{ + Pack: packB, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Length: uint(crypto.CiphertextLength(50)), Offset: 0, @@ -731,11 +733,11 @@ func TestRewriteFullPacks(t *testing.T) { } mi := index.NewMasterIndex() - rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packA, pack.Blobs{blobA.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) indexIDs := mi.IDs() @@ -750,6 +752,6 @@ func TestRewriteFullPacks(t *testing.T) { rtest.Equals(t, 2, len(afterRewrite)) rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs))) - rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle)) - rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle)) + rtest.Equals(t, []*pack.PackedBlob{blobA}, mi2.Lookup(blobA.Handle())) + rtest.Equals(t, []*pack.PackedBlob{blobB}, mi2.Lookup(blobB.Handle())) } diff --git a/internal/repository/index_list.go b/internal/repository/index_list.go index 18b40f114..c6d003539 100644 --- a/internal/repository/index_list.go +++ b/internal/repository/index_list.go @@ -28,7 +28,7 @@ func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.Load if ctx.Err() != nil { return ctx.Err() } - if !yield(IndexBlob{Handle: blob.BlobHandle}) { + if !yield(IndexBlob{Handle: blob.Handle()}) { return stopIteration } } diff --git a/internal/repository/index_list_test.go b/internal/repository/index_list_test.go index 2a9052879..9b7555709 100644 --- a/internal/repository/index_list_test.go +++ b/internal/repository/index_list_test.go @@ -26,8 +26,8 @@ func TestAllIndexBlobs(t *testing.T) { rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) fromMaster := restic.NewBlobSet() - rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - fromMaster.Insert(pb.BlobHandle) + rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + fromMaster.Insert(pb.Handle()) })) rtest.Equals(t, want, fromMaster) diff --git a/internal/repository/index_testutil_test.go b/internal/repository/index_testutil_test.go new file mode 100644 index 000000000..9e3a158ae --- /dev/null +++ b/internal/repository/index_testutil_test.go @@ -0,0 +1,18 @@ +package repository + +import ( + "github.com/restic/restic/internal/repository/pack" + "github.com/restic/restic/internal/restic" +) + +// BlobsInPack returns index entries for blobs stored in packID, sorted by offset. +func BlobsInPack(repo *Repository, packID restic.ID) pack.Blobs { + var blobs pack.Blobs + for pb := range repo.idx.Values() { + if pb.PackID().Equal(packID) { + blobs = append(blobs, pb.Blob) + } + } + blobs.Sort() + return blobs +} diff --git a/internal/repository/pack/blob.go b/internal/repository/pack/blob.go new file mode 100644 index 000000000..5ec8e6680 --- /dev/null +++ b/internal/repository/pack/blob.go @@ -0,0 +1,32 @@ +package pack + +import ( + "fmt" + + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/restic" +) + +// Blob is one part of a file or a tree with pack layout information. +type Blob struct { + restic.BlobHandle + Length uint + Offset uint + UncompressedLength uint +} + +func (b Blob) String() string { + return fmt.Sprintf("", + b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength) +} + +func (b Blob) DataLength() uint { + if b.UncompressedLength != 0 { + return b.UncompressedLength + } + return uint(crypto.PlaintextLength(int(b.Length))) +} + +func (b Blob) IsCompressed() bool { + return b.UncompressedLength != 0 +} diff --git a/internal/repository/pack/blobs.go b/internal/repository/pack/blobs.go new file mode 100644 index 000000000..04fe04d84 --- /dev/null +++ b/internal/repository/pack/blobs.go @@ -0,0 +1,15 @@ +package pack + +import ( + "cmp" + "slices" +) + +// Blobs is a list of blobs with pack layout information (offset, length, ...). +type Blobs []Blob + +func (b Blobs) Sort() { + slices.SortFunc(b, func(a, b Blob) int { + return cmp.Compare(a.Offset, b.Offset) + }) +} diff --git a/internal/repository/pack/blobs_test.go b/internal/repository/pack/blobs_test.go new file mode 100644 index 000000000..b345e482f --- /dev/null +++ b/internal/repository/pack/blobs_test.go @@ -0,0 +1,24 @@ +package pack + +import ( + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestBlobsSort(t *testing.T) { + blobs := Blobs{ + {Offset: 100}, + {Offset: 0}, + {Offset: 50}, + } + blobs.Sort() + rtest.Equals(t, uint(0), blobs[0].Offset) + rtest.Equals(t, uint(50), blobs[1].Offset) + rtest.Equals(t, uint(100), blobs[2].Offset) +} + +func TestBlobsSortNilSlice(t *testing.T) { + var blobs Blobs + blobs.Sort() +} diff --git a/internal/repository/pack/pack.go b/internal/repository/pack/pack.go index c065f4f65..b23043e7f 100644 --- a/internal/repository/pack/pack.go +++ b/internal/repository/pack/pack.go @@ -21,7 +21,7 @@ var ErrBroken = errors.New("packer cannot be used after a write error") // Packer is used to create a new Pack. type Packer struct { - blobs restic.Blobs + blobs []Blob bytes uint k *crypto.Key @@ -56,7 +56,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL return n, p.err } - c := restic.Blob{ + c := Blob{ BlobHandle: restic.BlobHandle{Type: t, ID: id}, Length: uint(n), Offset: p.bytes, @@ -65,7 +65,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL p.bytes += uint(n) p.blobs = append(p.blobs, c) - return n + CalculateEntrySize(c), nil + return n + CalculateEntrySize(c.IsCompressed()), nil } var entrySize = uint(binary.Size(restic.BlobType(0)) + 2*headerLengthSize + len(restic.ID{})) @@ -129,7 +129,7 @@ func (p *Packer) Finalize() error { return nil } -func verifyHeader(k *crypto.Key, header []byte, expected restic.Blobs) error { +func verifyHeader(k *crypto.Key, header []byte, expected []Blob) error { // do not offer a way to skip the pack header verification, as pack headers are usually small enough // to not result in a significant performance impact @@ -157,7 +157,7 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func makeHeader(blobs restic.Blobs) ([]byte, error) { +func makeHeader(blobs []Blob) ([]byte, error) { buf := make([]byte, 0, len(blobs)*int(entrySize)) for _, b := range blobs { @@ -232,7 +232,7 @@ func (p *Packer) HeaderFull() bool { } // Blobs returns the slice of blobs that have been written. -func (p *Packer) Blobs() restic.Blobs { +func (p *Packer) Blobs() Blobs { p.m.Lock() defer p.m.Unlock() @@ -348,7 +348,7 @@ func (e InvalidFileError) Error() string { // List returns the list of entries found in a pack file and the length of the // header (including header size and crypto overhead) -func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrSize uint32, err error) { +func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries Blobs, hdrSize uint32, err error) { buf, err := readHeader(rd, size) if err != nil { return nil, 0, err @@ -367,7 +367,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS } // might over allocate a bit if all blobs have EntrySize but only by a few percent - entries = make(restic.Blobs, 0, uint(len(buf))/plainEntrySize) + entries = make(Blobs, 0, uint(len(buf))/plainEntrySize) pos := uint(0) for len(buf) > 0 { @@ -385,7 +385,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS return entries, hdrSize, nil } -func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) { +func parseHeaderEntry(p []byte) (b Blob, size uint, err error) { l := uint(len(p)) size = plainEntrySize if l < plainEntrySize { @@ -420,17 +420,17 @@ func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) { return b, size, nil } -func CalculateEntrySize(blob restic.Blob) int { - if blob.UncompressedLength != 0 { +func CalculateEntrySize(compressed bool) int { + if compressed { return int(entrySize) } return int(plainEntrySize) } -func CalculateHeaderSize(blobs restic.Blobs) int { +func CalculateHeaderSize(blobs Blobs) int { size := headerSize for _, blob := range blobs { - size += CalculateEntrySize(blob) + size += CalculateEntrySize(blob.IsCompressed()) } return size } @@ -439,18 +439,19 @@ func CalculateHeaderSize(blobs restic.Blobs) int { // If onlyHdr is set to true, only the size of the header is returned // Note that this function only gives correct sizes, if there are no // duplicates in the index. -func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { +func Size(ctx context.Context, idx restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { packSize := make(map[restic.ID]int64) - err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) { - size, ok := packSize[blob.PackID] + err := idx.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + size, ok := packSize[packID] if !ok { size = headerSize } if !onlyHdr { - size += int64(blob.Length) + size += int64(blob.CiphertextLength()) } - packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob)) + packSize[packID] = size + int64(CalculateEntrySize(blob.IsCompressed())) }) return packSize, err diff --git a/internal/repository/pack/pack_internal_test.go b/internal/repository/pack/pack_internal_test.go index 186450e1c..d9a15298a 100644 --- a/internal/repository/pack/pack_internal_test.go +++ b/internal/repository/pack/pack_internal_test.go @@ -182,7 +182,7 @@ func TestReadRecords(t *testing.T) { func TestUnpackedVerification(t *testing.T) { // create random keys k := crypto.NewRandomKey() - blobs := restic.Blobs{ + blobs := []Blob{ { BlobHandle: restic.NewRandomBlobHandle(), Length: 42, diff --git a/internal/repository/pack/packedblob.go b/internal/repository/pack/packedblob.go new file mode 100644 index 000000000..e5d3e3588 --- /dev/null +++ b/internal/repository/pack/packedblob.go @@ -0,0 +1,19 @@ +package pack + +import "github.com/restic/restic/internal/restic" + +// PackedBlob is one index entry for a blob in a pack (may be duplicate across indexes). +type PackedBlob struct { + Pack restic.ID + Blob Blob +} + +func (pb *PackedBlob) PackID() restic.ID { return pb.Pack } + +func (pb *PackedBlob) Handle() restic.BlobHandle { return pb.Blob.BlobHandle } + +func (pb *PackedBlob) CiphertextLength() uint { return pb.Blob.Length } + +func (pb *PackedBlob) PlaintextLength() uint { return pb.Blob.DataLength() } + +func (pb *PackedBlob) IsCompressed() bool { return pb.Blob.IsCompressed() } diff --git a/internal/repository/prune.go b/internal/repository/prune.go index d534d20e3..becbc99d2 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -142,11 +142,12 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed if len(plan.repackPacks) != 0 { // when repacking, we do not want to keep blobs which are // already contained in kept packs, so delete them from keepBlobs - err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) { + err := repo.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + if plan.removePacks.Has(packID) || plan.repackPacks.Has(packID) { return } - keepBlobs.Delete(blob.BlobHandle) + keepBlobs.Delete(blob.Handle()) }) if err != nil { return nil, err @@ -179,8 +180,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i // iterate over all blobs in index to find out which blobs are duplicates // The counter in usedBlobs describes how many instances of the blob exist in the repository index // Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist - err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle + err := idx.ListBlobs(ctx, func(blob restic.PackBlob) { + bh := blob.Handle() count, ok := usedBlobs.Get(bh) if ok { if count < math.MaxUint8 { @@ -229,22 +230,24 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i hasDuplicates := false // iterate over all blobs in index to generate packInfo - err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - ip := indexPack[blob.PackID] + err = idx.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + h := blob.Handle() + + ip := indexPack[packID] // Set blob type if not yet set if ip.tpe == restic.NumBlobTypes { - ip.tpe = blob.Type + ip.tpe = h.Type } // mark mixed packs with "Invalid blob type" - if ip.tpe != blob.Type { + if ip.tpe != h.Type { ip.tpe = restic.InvalidBlob } - bh := blob.BlobHandle - size := uint64(blob.Length) - dupCount, _ := usedBlobs.Get(bh) + size := uint64(blob.CiphertextLength()) + dupCount, _ := usedBlobs.Get(h) switch { case dupCount >= 2: hasDuplicates = true @@ -273,7 +276,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i ip.uncompressed = true } // update indexPack - indexPack[blob.PackID] = ip + indexPack[packID] = ip }) if err != nil { return nil, nil, err @@ -287,8 +290,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i // - if there are no used blobs in a pack, possibly mark duplicates as "unused" if hasDuplicates { // iterate again over all blobs in index (this is pretty cheap, all in-mem) - err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle + err = idx.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + bh := blob.Handle() count, ok := usedBlobs.Get(bh) // skip non-duplicate, aka. normal blobs // count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining @@ -296,8 +300,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i return } - ip := indexPack[blob.PackID] - size := uint64(blob.Length) + ip := indexPack[packID] + size := uint64(blob.CiphertextLength()) switch { case ip.usedBlobs > 0, ip.duplicateBlobs == ip.unusedBlobs, count == 0: // other used blobs in pack, only duplicate blobs or "last" occurrence -> transition to used @@ -325,7 +329,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i usedBlobs.Set(bh, count) } // update indexPack - indexPack[blob.PackID] = ip + indexPack[packID] = ip }) if err != nil { return nil, nil, err diff --git a/internal/repository/repack.go b/internal/repository/repack.go index 2793348b8..d8e351089 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -7,6 +7,8 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" @@ -30,7 +32,7 @@ type LogFunc func(msg string, args ...interface{}) // blobs have been processed. func CopyBlobs( ctx context.Context, - repo restic.Repository, + repo *Repository, dstRepo restic.Repository, dstUploader restic.BlobSaverWithAsync, packs restic.IDSet, @@ -55,7 +57,7 @@ func CopyBlobs( func repack( ctx context.Context, - repo restic.Repository, + repo *Repository, dstRepo restic.Repository, uploader restic.BlobSaverWithAsync, packs restic.IDSet, @@ -80,11 +82,11 @@ func repack( } var keepMutex sync.Mutex - downloadQueue := make(chan restic.PackBlobs) + downloadQueue := make(chan index.PackBlobs) wg.Go(func() error { defer close(downloadQueue) - for pbs := range repo.ListPacksFromIndex(wgCtx, packs) { - var packBlobs restic.Blobs + for pbs := range repo.listPacksFromIndex(wgCtx, packs) { + var packBlobs pack.Blobs keepMutex.Lock() // filter out unnecessary blobs for _, entry := range pbs.Blobs { @@ -96,7 +98,7 @@ func repack( keepMutex.Unlock() select { - case downloadQueue <- restic.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}: + case downloadQueue <- index.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}: case <-wgCtx.Done(): return wgCtx.Err() } @@ -106,7 +108,7 @@ func repack( worker := func() error { for t := range downloadQueue { - err := repo.LoadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + err := repo.loadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { // a required blob couldn't be retrieved return err diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index c490759ba..909d0af91 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -86,13 +86,12 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa blobs := restic.NewBlobSet() err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - entries, err := repo.ListPack(context.TODO(), id, size) + handles, err := repo.ListPackHandles(context.TODO(), id, size) if err != nil { t.Fatalf("error listing pack %v: %v", id, err) } - for _, entry := range entries { - h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} + for _, h := range handles { if blobs.Has(h) { t.Errorf("ignoring duplicate blob %v", h) return nil @@ -100,9 +99,9 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa blobs.Insert(h) if random.Float32() <= p { - list1.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) + list1.Insert(h) } else { - list2.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) + list2.Insert(h) } } return nil @@ -142,14 +141,14 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe } for _, pb := range list { - packs.Insert(pb.PackID) + packs.Insert(pb.PackID()) } } return packs } -func repack(t *testing.T, repo restic.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) { +func repack(t *testing.T, repo *repository.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) { rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, nil, nil) })) @@ -222,8 +221,8 @@ func testRepack(t *testing.T, version uint) { pb := list[0] - if removePacks.Has(pb.PackID) { - t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID) + if removePacks.Has(pb.PackID()) { + t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID()) } } @@ -238,21 +237,28 @@ func TestRepackCopy(t *testing.T) { repository.TestAllVersions(t, testRepackCopy) } -type oneConnectionRepo struct { - restic.Repository +// oneConnectionBackend limits concurrent backend operations to test repack with +// the minimum connection count required by CopyBlobs. +type oneConnectionBackend struct { + backend.Backend } -func (r oneConnectionRepo) Connections() uint { - return 1 +func (be *oneConnectionBackend) Properties() backend.Properties { + p := be.Backend.Properties() + p.Connections = 1 + return p +} + +func (be *oneConnectionBackend) Unwrap() backend.Backend { + return be.Backend } func testRepackCopy(t *testing.T, version uint) { - repo, _, _ := repository.TestRepositoryWithVersion(t, version) - dstRepo, _, _ := repository.TestRepositoryWithVersion(t, version) - // test with minimal possible connection count - repoWrapped := &oneConnectionRepo{repo} - dstRepoWrapped := &oneConnectionRepo{dstRepo} + repo, _ := repository.TestRepositoryWithBackend(t, &oneConnectionBackend{Backend: repository.TestBackend(t)}, version, repository.Options{}) + dstRepo, _ := repository.TestRepositoryWithBackend(t, &oneConnectionBackend{Backend: repository.TestBackend(t)}, version, repository.Options{}) + rtest.Equals(t, repo.Connections(), 1) + rtest.Equals(t, dstRepo.Connections(), 1) seed := time.Now().UnixNano() random := rand.New(rand.NewSource(seed)) @@ -265,8 +271,8 @@ func testRepackCopy(t *testing.T, version uint) { _, keepBlobs := selectBlobs(t, random, repo, 0.2) copyPacks := findPacksForBlobs(t, repo, keepBlobs) - rtest.OK(t, repoWrapped.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { - return repository.CopyBlobs(ctx, repoWrapped, dstRepoWrapped, uploader, copyPacks, keepBlobs, nil, nil) + rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { + return repository.CopyBlobs(ctx, repo, dstRepo, uploader, copyPacks, keepBlobs, nil, nil) })) rebuildAndReloadIndex(t, dstRepo) diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index 5059aad04..1c27acf5b 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -6,6 +6,7 @@ import ( "io" "slices" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" ) @@ -23,7 +24,7 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { // examine all data the indexes have for the pack file - for b := range repo.ListPacksFromIndex(ctx, ids) { + for b := range repo.listPacksFromIndex(ctx, ids) { indexBlobs := b.Blobs err := reuploadBlobsFromPack(ctx, repo, b.PackID, indexBlobs, printer, uploader) if err != nil { @@ -71,12 +72,12 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe return nil } -func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]restic.Blobs, error) { - packToBlobs := make(map[restic.ID]restic.Blobs) +func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]pack.Blobs, error) { + packToBlobs := make(map[restic.ID]pack.Blobs) err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { if ids.Has(id) { - blobs, err := repo.ListPack(ctx, id, size) + blobs, err := repo.listPack(ctx, id, size) if err != nil { return nil } @@ -90,8 +91,8 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe return packToBlobs, nil } -func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs restic.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error { - err := repo.LoadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { +func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs pack.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error { + err := repo.loadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { printer.E("failed to load blob %v: %v", blob.ID, err) return nil diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index 558889e0d..cbf71bf16 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -16,8 +16,8 @@ import ( func listBlobs(repo restic.Repository) restic.BlobSet { blobs := restic.NewBlobSet() - _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - blobs.Insert(pb.BlobHandle) + _ = repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + blobs.Insert(pb.Handle()) }) return blobs } @@ -66,11 +66,10 @@ func testRepairBrokenPack(t *testing.T, version uint) { // find blob that starts at offset 0 var damagedBlob restic.BlobHandle - for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { - for _, blob := range blobs.Blobs { - if blob.Offset == 0 { - damagedBlob = blob.BlobHandle - } + for _, blob := range repository.BlobsInPack(repo, damagedID) { + if blob.Offset == 0 { + damagedBlob = blob.BlobHandle + break } } @@ -89,11 +88,11 @@ func testRepairBrokenPack(t *testing.T, version uint) { // all blobs in the file are broken damagedBlobs := restic.NewBlobSet() - for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { - for _, blob := range blobs.Blobs { - damagedBlobs.Insert(blob.BlobHandle) + rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + if pb.PackID().Equal(damagedID) { + damagedBlobs.Insert(pb.Handle()) } - } + })) return restic.NewIDSet(damagedID), damagedBlobs }, }, { diff --git a/internal/repository/repository.go b/internal/repository/repository.go index eb6c0958b..828559f3b 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -213,7 +213,7 @@ type haver interface { } // sortCachedPacksFirst moves all cached pack files to the front of blobs. -func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { +func sortCachedPacksFirst(cache haver, blobs []*pack.PackedBlob) { if cache == nil { return } @@ -224,10 +224,10 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { } cached := blobs[:0] - noncached := make([]restic.PackedBlob, 0, len(blobs)/2) + noncached := make([]*pack.PackedBlob, 0, len(blobs)/2) for _, blob := range blobs { - if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { + if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID().String()}) { cached = append(cached, blob) continue } @@ -256,7 +256,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. if err != nil { if r.cache != nil { for _, blob := range blobs { - h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()} // ignore errors as there's not much we can do here _ = r.cache.Forget(h) } @@ -267,28 +267,28 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. return buf, err } -func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, buf []byte) ([]byte, error) { +func (r *Repository) loadBlob(ctx context.Context, blobs []*pack.PackedBlob, buf []byte) ([]byte, error) { var lastError error for _, blob := range blobs { - debug.Log("blob %v found: %v", blob.BlobHandle, blob) + debug.Log("blob %v found: %v", blob.Handle(), blob) // load blob from pack - h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()} switch { - case cap(buf) < int(blob.Length): - buf = make([]byte, blob.Length) - case len(buf) != int(blob.Length): - buf = buf[:blob.Length] + case cap(buf) < int(blob.Blob.Length): + buf = make([]byte, blob.Blob.Length) + case len(buf) != int(blob.Blob.Length): + buf = buf[:blob.Blob.Length] } - _, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) + _, err := backend.ReadAt(ctx, r.be, h, int64(blob.Blob.Offset), buf) if err != nil { debug.Log("error loading blob %v: %v", blob, err) lastError = err continue } - it := newPackBlobIterator(blob.PackID, newByteReader(buf), blob.Offset, restic.Blobs{blob.Blob}, r.key, r.getZstdDecoder()) + it := newPackBlobIterator(blob.PackID(), newByteReader(buf), blob.Blob.Offset, pack.Blobs{blob.Blob}, r.key, r.getZstdDecoder()) pbv, err := it.Next() if err == nil { @@ -314,7 +314,7 @@ func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, bu return nil, lastError } - return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs)) + return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].Handle(), len(blobs)) } func (r *Repository) getZstdEncoder() *zstd.Encoder { @@ -672,8 +672,13 @@ func (r *Repository) Connections() uint { return r.be.Properties().Connections } -func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackedBlob { - return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) +func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob { + entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) + out := make([]restic.PackBlob, len(entries)) + for i, e := range entries { + out[i] = e + } + return out } // LookupBlobSize returns the size of blob id. Also returns pending blobs. @@ -683,7 +688,7 @@ func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bo // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. -func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error { +func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) error { for blob := range r.idx.Values() { if ctx.Err() != nil { return ctx.Err() @@ -693,7 +698,8 @@ func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) return nil } -func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { +// listPacksFromIndex returns index entries for the given packs, grouped by pack file. +func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan index.PackBlobs { return r.idx.ListPacks(ctx, packs) } @@ -783,7 +789,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest // a worker receives an pack ID from ch, reads the pack contents, and adds them to idx worker := func() error { for fi := range ch { - entries, err := r.ListPack(wgCtx, fi.ID, fi.Size) + entries, err := r.listPack(wgCtx, fi.ID, fi.Size) if err != nil { debug.Log("unable to list pack file %v", fi.ID.Str()) m.Lock() @@ -964,8 +970,8 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic }) } -// ListPack returns the list of blobs saved in the pack id. -func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) { +// listPack returns blob entries from the pack file header including offsets. +func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (pack.Blobs, error) { h := backend.Handle{Type: restic.PackFile, Name: id.String()} entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) @@ -978,7 +984,20 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re // retry on error entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) } - return entries, err + return pack.Blobs(entries), err +} + +// ListPackHandles returns the blob handles stored in the pack file header. +func (r *Repository) ListPackHandles(ctx context.Context, id restic.ID, size int64) ([]restic.BlobHandle, error) { + blobs, err := r.listPack(ctx, id, size) + if err != nil { + return nil, err + } + handles := make([]restic.BlobHandle, len(blobs)) + for i, blob := range blobs { + handles[i] = blob.BlobHandle + } + return handles, nil } // Delete calls backend.Delete() if implemented, and returns an error @@ -1054,11 +1073,37 @@ const maxUnusedRange = 1 * 1024 * 1024 // handleBlobFn is called at most once for each blob. If the callback returns an error, // then LoadBlobsFromPack will abort and not retry it. The buf passed to the callback is only valid within // this specific call. The callback must not keep a reference to buf. -func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + blobs, err := r.blobsInPack(packID, handles) + if err != nil { + return err + } + return r.loadBlobsFromPack(ctx, packID, blobs, handleBlobFn) +} + +func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (pack.Blobs, error) { + blobs := make(pack.Blobs, 0, len(handles)) + for _, h := range handles { + found := false + for _, pb := range r.idx.Lookup(h) { + if pb.PackID().Equal(packID) { + blobs = append(blobs, pb.Blob) + found = true + break + } + } + if !found { + return nil, errors.Errorf("blob %v not found in pack %v", h, packID) + } + } + return blobs, nil +} + +func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return streamPack(ctx, r.be.Load, r.LoadBlob, r.getZstdDecoder(), r.key, packID, blobs, handleBlobFn) } -func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do return nil @@ -1101,7 +1146,7 @@ func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn return streamPackPart(ctx, beLoad, loadBlobFn, dec, key, packID, blobs[lowerIdx:], handleBlobFn) } -func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: blobs[0].Type.IsMetadata()} dataStart := blobs[0].Offset @@ -1211,7 +1256,7 @@ type packBlobIterator struct { rd discardReader currentOffset uint - blobs restic.Blobs + blobs pack.Blobs key *crypto.Key dec *zstd.Decoder @@ -1227,7 +1272,7 @@ type packBlobValue struct { var errPackEOF = errors.New("reached EOF of pack file") func newPackBlobIterator(packID restic.ID, rd discardReader, currentOffset uint, - blobs restic.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator { + blobs pack.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator { return &packBlobIterator{ packID: packID, rd: rd, diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index 31aed6f64..226c3c237 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -17,6 +17,7 @@ import ( "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,7 +28,7 @@ func (c mapcache) Has(h backend.Handle) bool { return c[h] } func TestSortCachedPacksFirst(t *testing.T) { var ( - blobs, sorted [100]restic.PackedBlob + blobs, sorted [100]*pack.PackedBlob cache = make(mapcache) r = rand.New(rand.NewSource(1261)) @@ -36,7 +37,7 @@ func TestSortCachedPacksFirst(t *testing.T) { for i := 0; i < len(blobs); i++ { var id restic.ID r.Read(id[:]) - blobs[i] = restic.PackedBlob{PackID: id} + blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}} if i%3 == 0 { h := backend.Handle{Name: id.String(), Type: backend.PackFile} @@ -46,8 +47,8 @@ func TestSortCachedPacksFirst(t *testing.T) { copy(sorted[:], blobs[:]) sort.SliceStable(sorted[:], func(i, j int) bool { - hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()} - hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()} + hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID().String()} + hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID().String()} return cache.Has(hi) && !cache.Has(hj) }) @@ -59,7 +60,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { const nblobs = 512 // Corresponds to a file of ca. 2GB. var ( - blobs [nblobs]restic.PackedBlob + blobs [nblobs]*pack.PackedBlob cache = make(mapcache) r = rand.New(rand.NewSource(1261)) ) @@ -67,7 +68,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { for i := 0; i < nblobs; i++ { var id restic.ID r.Read(id[:]) - blobs[i] = restic.PackedBlob{PackID: id} + blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}} if i%3 == 0 { h := backend.Handle{Name: id.String(), Type: backend.PackFile} @@ -75,7 +76,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { } } - var cpy [nblobs]restic.PackedBlob + var cpy [nblobs]*pack.PackedBlob b.ReportAllocs() b.ResetTimer() @@ -96,7 +97,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { idx := index.NewIndex() for i := 0; i < 5000; i++ { - idx.StorePack(restic.NewRandomID(), restic.Blobs{ + idx.StorePack(restic.NewRandomID(), pack.Blobs{ { BlobHandle: restic.NewRandomBlobHandle(), Length: 1234, @@ -133,7 +134,7 @@ func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (* } // buildPackfileWithoutHeader returns a manually built pack file without a header. -func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs restic.Blobs, packfile []byte) { +func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs pack.Blobs, packfile []byte) { opts := []zstd.EOption{ // Set the compression level configured. zstd.WithEncoderLevel(zstd.SpeedDefault), @@ -173,7 +174,7 @@ func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) ciphertextLength := after - before - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: id, @@ -280,19 +281,19 @@ func testStreamPack(t *testing.T, version uint) { // first, test regular usage t.Run("regular", func(t *testing.T) { tests := []struct { - blobs restic.Blobs + blobs pack.Blobs calls int shortFirstLoad bool }{ {packfileBlobs[1:2], 1, false}, {packfileBlobs[2:5], 1, false}, {packfileBlobs[2:8], 1, false}, - {restic.Blobs{ + {pack.Blobs{ packfileBlobs[0], packfileBlobs[4], packfileBlobs[2], }, 1, false}, - {restic.Blobs{ + {pack.Blobs{ packfileBlobs[0], packfileBlobs[len(packfileBlobs)-1], }, 2, false}, @@ -341,12 +342,12 @@ func testStreamPack(t *testing.T, version uint) { // next, test invalid uses, which should return an error t.Run("invalid", func(t *testing.T) { tests := []struct { - blobs restic.Blobs + blobs pack.Blobs err string }{ { // pass one blob several times - blobs: restic.Blobs{ + blobs: pack.Blobs{ packfileBlobs[3], packfileBlobs[8], packfileBlobs[3], @@ -357,7 +358,7 @@ func testStreamPack(t *testing.T, version uint) { { // pass something that's not a valid blob in the current pack file - blobs: restic.Blobs{ + blobs: pack.Blobs{ { Offset: 123, Length: 20000, @@ -368,7 +369,7 @@ func testStreamPack(t *testing.T, version uint) { { // pass a blob that's too small - blobs: restic.Blobs{ + blobs: pack.Blobs{ { Offset: 123, Length: 10, @@ -523,7 +524,7 @@ func TestStreamPackFallback(t *testing.T) { plaintext := rtest.Random(800, 42) blobID := restic.Hash(plaintext) - blobs := restic.Blobs{ + blobs := pack.Blobs{ { Length: uint(crypto.CiphertextLength(len(plaintext))), Offset: 0, diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index dc4febaaf..083768a80 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -233,7 +233,7 @@ func TestLoadBlobBroken(t *testing.T) { data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil) rtest.OK(t, err) rtest.Assert(t, bytes.Equal(buf, data), "data mismatch") - pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID() rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached") } @@ -422,11 +422,12 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { rtest.OK(t, err) for pb := range idx.Values() { - if _, ok := packEntries[pb.PackID]; !ok { - packEntries[pb.PackID] = make(map[restic.ID]struct{}) + packID := pb.PackID() + if _, ok := packEntries[packID]; !ok { + packEntries[packID] = make(map[restic.ID]struct{}) } - packEntries[pb.PackID][id] = struct{}{} + packEntries[packID][id] = struct{}{} } return nil }) @@ -467,7 +468,7 @@ func TestListPack(t *testing.T) { repo.UseCache(c, t.Logf) // Forcibly cache pack file - packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID() rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil })) // Get size to list pack @@ -479,11 +480,11 @@ func TestListPack(t *testing.T) { return nil })) - blobs, err := repo.ListPack(context.TODO(), packID, size) + handles, err := repo.ListPackHandles(context.TODO(), packID, size) rtest.OK(t, err) - rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs) + rtest.Assert(t, len(handles) == 1 && handles[0].ID == id, "unexpected blobs in pack: %v", handles) - rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as ListPack does not set IsMetadata in the backend.Handle") + rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as listPack does not set IsMetadata in the backend.Handle") } func TestNoDoubleInit(t *testing.T) { diff --git a/internal/restic/blob.go b/internal/restic/blob.go index ba9277aac..5fad7b3f6 100644 --- a/internal/restic/blob.go +++ b/internal/restic/blob.go @@ -1,50 +1,22 @@ package restic import ( - "cmp" "fmt" - "slices" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" ) -// Blob is one part of a file or a tree. -type Blob struct { - BlobHandle - Length uint - Offset uint - UncompressedLength uint -} - -func (b Blob) String() string { - return fmt.Sprintf("", - b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength) -} - -func (b Blob) DataLength() uint { - if b.UncompressedLength != 0 { - return b.UncompressedLength - } - return uint(crypto.PlaintextLength(int(b.Length))) -} - -func (b Blob) IsCompressed() bool { - return b.UncompressedLength != 0 -} - -type Blobs []Blob - -func (b Blobs) Sort() { - slices.SortFunc(b, func(a, b Blob) int { - return cmp.Compare(a.Offset, b.Offset) - }) -} - -// PackedBlob is a blob stored within a file. -type PackedBlob struct { - Blob - PackID ID +// PackBlob is one index entry for a blob in a pack file. +// The interface intentionally omits the offset at which a blob is stored in the pack. +// This ensures that pack file internals are not leaked. +type PackBlob interface { + PackID() ID + Handle() BlobHandle + // CiphertextLength is the encrypted size stored in the pack. + CiphertextLength() uint + // PlaintextLength is the size after decryption/decompression. + PlaintextLength() uint + IsCompressed() bool } // BlobHandle identifies a blob of a given type. diff --git a/internal/restic/blob_test.go b/internal/restic/blob_test.go index 36e045792..951872250 100644 --- a/internal/restic/blob_test.go +++ b/internal/restic/blob_test.go @@ -3,8 +3,6 @@ package restic import ( "encoding/json" "testing" - - rtest "github.com/restic/restic/internal/test" ) var blobTypeJSON = []struct { @@ -41,20 +39,3 @@ func TestBlobTypeJSON(t *testing.T) { } } } - -func TestBlobsSort(t *testing.T) { - blobs := Blobs{ - {Offset: 100}, - {Offset: 0}, - {Offset: 50}, - } - blobs.Sort() - rtest.Equals(t, uint(0), blobs[0].Offset) - rtest.Equals(t, uint(50), blobs[1].Offset) - rtest.Equals(t, uint(100), blobs[2].Offset) -} - -func TestBlobsSortNilSlice(t *testing.T) { - var blobs Blobs - blobs.Sort() -} diff --git a/internal/restic/repository.go b/internal/restic/repository.go index f8b5cef33..b3e688271 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -24,19 +24,18 @@ type Repository interface { LoadIndex(ctx context.Context, p TerminalCounterFactory) error - LookupBlob(t BlobType, id ID) []PackedBlob + LookupBlob(t BlobType, id ID) []PackBlob LookupBlobSize(t BlobType, id ID) (size uint, exists bool) NewAssociatedBlobSet() AssociatedBlobSet // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. - ListBlobs(ctx context.Context, fn func(PackedBlob)) error - ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs - // ListPack returns the list of blobs saved in the pack id. - ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error) + ListBlobs(ctx context.Context, fn func(PackBlob)) error + // ListPackHandles returns the blob handles stored in the pack file header. + ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error) LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error) - LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error + LoadBlobsFromPack(ctx context.Context, packID ID, blobs []BlobHandle, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error // WithUploader starts the necessary workers to upload new blobs. Once the callback returns, // the workers are stopped and the index is written to the repository. The callback must use @@ -125,11 +124,6 @@ type SaverRemoverUnpacked[FT FileTypes] interface { RemoverUnpacked[FT] } -type PackBlobs struct { - PackID ID - Blobs Blobs -} - type TerminalCounterFactory interface { // NewCounterTerminalOnly returns a new progress counter that is only shown if stdout points to a // terminal. It is not shown if --quiet or --json is specified. @@ -153,7 +147,7 @@ type Unpacked[FT FileTypes] interface { } type ListBlobser interface { - ListBlobs(ctx context.Context, fn func(PackedBlob)) error + ListBlobs(ctx context.Context, fn func(PackBlob)) error } type BlobLoader interface { diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 63dc3d6e9..3ec312a2c 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -42,12 +42,12 @@ type packInfo struct { files map[*fileInfo]struct{} // set of files that use blobs from this pack } -type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error +type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error type startWarmupFn func(context.Context, restic.IDSet) (restic.WarmupJob, error) // fileRestorer restores set of files type fileRestorer struct { - idx func(restic.BlobType, restic.ID) []restic.PackedBlob + idx func(restic.BlobType, restic.ID) []restic.PackBlob blobsLoader blobsLoaderFn startWarmup startWarmupFn @@ -68,7 +68,7 @@ type fileRestorer struct { func newFileRestorer(dst string, blobsLoader blobsLoaderFn, - idx func(restic.BlobType, restic.ID) []restic.PackedBlob, + idx func(restic.BlobType, restic.ID) []restic.PackBlob, connections uint, sparse bool, allowRecursiveDelete bool, @@ -102,7 +102,7 @@ func (r *fileRestorer) targetPath(location string) string { return filepath.Join(r.dst, location) } -func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int, fileOffset int64)) error { +func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(blob restic.PackBlob, idx int, fileOffset int64)) error { if len(blobIDs) == 0 { return nil } @@ -114,8 +114,8 @@ func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID return errors.Errorf("Unknown blob %s", blobID.String()) } pb := packs[0] - fn(pb.PackID, pb.Blob, i, fileOffset) - fileOffset += int64(pb.DataLength()) + fn(pb, i, fileOffset) + fileOffset += int64(pb.PlaintextLength()) } return nil @@ -143,14 +143,15 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { file.blobs = packsMap } restoredBlobs := false - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { + err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) { + packID := blob.PackID() if !file.state.HasMatchingBlob(idx) { if largeFile { - packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset}) + packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.Handle().ID, offset: fileOffset}) } restoredBlobs = true } else { - r.reportBlobProgress(file, uint64(blob.DataLength())) + r.reportBlobProgress(file, uint64(blob.PlaintextLength())) // completely ignore blob return } @@ -164,7 +165,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { packOrder = append(packOrder, packID) } pack.files[file] = struct{}{} - if blob.ID.Equal(r.zeroChunk) { + if blob.Handle().ID.Equal(r.zeroChunk) { file.sparse = r.sparse } }) @@ -261,14 +262,14 @@ func (r *fileRestorer) truncateFileToSize(location string, size int64) error { type blobToFileOffsetsMapping map[restic.ID]struct { files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file - blob restic.Blob + blob restic.BlobHandle } func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // calculate blob->[]files->[]offsets mappings blobs := make(blobToFileOffsetsMapping) for file := range pack.files { - addBlob := func(blob restic.Blob, fileOffset int64) { + addBlob := func(blob restic.BlobHandle, fileOffset int64) { blobInfo, ok := blobs[blob.ID] if !ok { blobInfo.files = make(map[*fileInfo][]int64) @@ -278,9 +279,9 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { blobInfo.files[file] = append(blobInfo.files[file], fileOffset) } if fileBlobs, ok := file.blobs.(restic.IDs); ok { - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { - if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) { - addBlob(blob, fileOffset) + err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) { + if blob.PackID().Equal(pack.id) && !file.state.HasMatchingBlob(idx) { + addBlob(blob.Handle(), fileOffset) } }) if err != nil { @@ -291,8 +292,8 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { for _, blob := range packsMap[pack.id] { idxPacks := r.idx(restic.DataBlob, blob.id) for _, idxPack := range idxPacks { - if idxPack.PackID.Equal(pack.id) { - addBlob(idxPack.Blob, blob.offset) + if idxPack.PackID().Equal(pack.id) { + addBlob(idxPack.Handle(), blob.offset) break } } @@ -324,7 +325,7 @@ func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlob // only report error for not yet processed blobs affectedFiles := make(map[*fileInfo]struct{}) for _, entry := range blobs { - if processedBlobs.Has(entry.blob.BlobHandle) { + if processedBlobs.Has(entry.blob) { continue } for file := range entry.files { @@ -343,7 +344,7 @@ func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlob func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { - blobList := make(restic.Blobs, 0, len(blobs)) + blobList := make([]restic.BlobHandle, 0, len(blobs)) for _, entry := range blobs { blobList = append(blobList, entry.blob) } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index 120288378..d517a741d 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -2,9 +2,11 @@ package restorer import ( "bytes" + "cmp" "context" "fmt" "os" + "slices" "testing" "github.com/restic/restic/internal/errors" @@ -28,11 +30,32 @@ type TestWarmupJob struct { waitCalled bool } +type testPackBlob struct { + packID restic.ID + handle restic.BlobHandle + offset uint + ciphertext uint + plaintext uint + compressed bool +} + +var _ restic.PackBlob = (*testPackBlob)(nil) + +func (pb *testPackBlob) PackID() restic.ID { return pb.packID } + +func (pb *testPackBlob) Handle() restic.BlobHandle { return pb.handle } + +func (pb *testPackBlob) CiphertextLength() uint { return pb.ciphertext } + +func (pb *testPackBlob) PlaintextLength() uint { return pb.plaintext } + +func (pb *testPackBlob) IsCompressed() bool { return pb.compressed } + type TestRepo struct { packsIDToData map[restic.ID][]byte // blobs and files - blobs map[restic.ID][]restic.PackedBlob + blobs map[restic.ID][]restic.PackBlob files []*fileInfo filesPathToContent map[string]string @@ -42,7 +65,7 @@ type TestRepo struct { loader blobsLoaderFn } -func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackedBlob { +func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackBlob { packs := i.blobs[id] return packs } @@ -67,10 +90,16 @@ func (job *TestWarmupJob) Wait(_ context.Context) error { } func newTestRepo(content []TestFile) *TestRepo { + type packBlobLayout struct { + offset uint + ciphertext uint + plaintext uint + compressed bool + } type Pack struct { name string data []byte - blobs map[restic.ID]restic.Blob + blobs map[restic.ID]packBlobLayout } packs := make(map[string]Pack) filesPathToContent := make(map[string]string) @@ -84,21 +113,19 @@ func newTestRepo(content []TestFile) *TestRepo { var pack Pack var found bool if pack, found = packs[blob.pack]; !found { - pack = Pack{name: blob.pack, blobs: make(map[restic.ID]restic.Blob)} + pack = Pack{name: blob.pack, blobs: make(map[restic.ID]packBlobLayout)} } // calculate blob id and add to the pack as necessary blobID := restic.Hash([]byte(blob.data)) if _, found := pack.blobs[blobID]; !found { blobData := []byte(blob.data) - pack.blobs[blobID] = restic.Blob{ - BlobHandle: restic.BlobHandle{ - Type: restic.DataBlob, - ID: blobID, - }, - Length: uint(len(blobData)), - UncompressedLength: uint(len(blobData)), - Offset: uint(len(pack.data)), + n := uint(len(blobData)) + pack.blobs[blobID] = packBlobLayout{ + offset: uint(len(pack.data)), + ciphertext: n, + plaintext: n, + compressed: true, } pack.data = append(pack.data, blobData...) } @@ -108,14 +135,19 @@ func newTestRepo(content []TestFile) *TestRepo { filesPathToContent[file.name] = content } - blobs := make(map[restic.ID][]restic.PackedBlob) + blobs := make(map[restic.ID][]restic.PackBlob) packsIDToData := make(map[restic.ID][]byte) for _, pack := range packs { packID := restic.Hash(pack.data) packsIDToData[packID] = pack.data - for blobID, blob := range pack.blobs { - blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID}) + for blobID, layout := range pack.blobs { + blobs[blobID] = append(blobs[blobID], &testPackBlob{ + packID: packID, + handle: restic.BlobHandle{Type: restic.DataBlob, ID: blobID}, + offset: layout.offset, ciphertext: layout.ciphertext, + plaintext: layout.plaintext, compressed: layout.compressed, + }) } } @@ -135,24 +167,30 @@ func newTestRepo(content []TestFile) *TestRepo { filesPathToContent: filesPathToContent, warmupJobs: []*TestWarmupJob{}, } - repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { - blobs = append(restic.Blobs{}, blobs...) - blobs.Sort() - - for _, blob := range blobs { + repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + entries := make([]*testPackBlob, 0, len(handles)) + for _, h := range handles { found := false - for _, e := range repo.blobs[blob.ID] { - if packID == e.PackID { + for _, e := range repo.blobs[h.ID] { + if packID == e.PackID() { + entries = append(entries, e.(*testPackBlob)) found = true - buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] - err := handleBlobFn(e.BlobHandle, buf, nil) - if err != nil { - return err - } + break } } if !found { - return fmt.Errorf("missing blob: %v", blob) + return fmt.Errorf("missing blob: %v", h) + } + } + slices.SortFunc(entries, func(a, b *testPackBlob) int { + return cmp.Compare(a.offset, b.offset) + }) + + for _, e := range entries { + buf := repo.packsIDToData[packID][e.offset : e.offset+e.ciphertext] + err := handleBlobFn(e.handle, buf, nil) + if err != nil { + return err } } return nil @@ -313,7 +351,7 @@ func TestErrorRestoreFiles(t *testing.T) { loadError := errors.New("load error") // loader always returns an error - repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return loadError } @@ -346,9 +384,9 @@ func TestFatalDownloadError(t *testing.T) { repo := newTestRepo(content) loader := repo.loader - repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { ctr := 0 - return loader(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + return loader(ctx, packID, handles, func(blob restic.BlobHandle, buf []byte, err error) error { if ctr < 2 { ctr++ return handleBlobFn(blob, buf, err)