diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 9fde55560..5069cfe7c 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -525,22 +525,23 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc // remember which packs were found in the index indexPackIDs := make(map[string]struct{}) - err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) { - idStr := pb.PackID.String() + err := f.repo.ListBlobs(wctx, func(pb restic.PackBlob) { + packID := pb.PackID() + idStr := packID.String() // keep entry in packIDs as Each() returns individual index entries matchingID := false if _, ok := packIDs[idStr]; ok { matchingID = true } else { - if _, ok := packIDs[pb.PackID.Str()]; ok { + if _, ok := packIDs[packID.Str()]; ok { // expand id - delete(packIDs, pb.PackID.Str()) + delete(packIDs, packID.Str()) packIDs[idStr] = struct{}{} matchingID = true } } if matchingID { - f.blobIDs[pb.ID.String()] = struct{}{} + f.blobIDs[pb.Handle().ID.String()] = struct{}{} indexPackIDs[idStr] = struct{}{} } }) diff --git a/cmd/restic/cmd_find_integration_test.go b/cmd/restic/cmd_find_integration_test.go index f46d63cc5..5935f96b8 100644 --- a/cmd/restic/cmd_find_integration_test.go +++ b/cmd/restic/cmd_find_integration_test.go @@ -179,9 +179,10 @@ func TestFindPackfile(t *testing.T) { packID := restic.ID{} done := false - err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) { - if !done && pb.Type == restic.TreeBlob { - packID = pb.PackID + err = repo.ListBlobs(ctx, func(pb restic.PackBlob) { + h := pb.Handle() + if !done && h.Type == restic.TreeBlob { + packID = pb.PackID() done = true } }) @@ -236,12 +237,12 @@ func TestFindPackID(t *testing.T) { // load Index rtest.OK(t, repo.LoadIndex(ctx, nil)) // go through all index entries and collect data and tree packfile(s) - rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - switch blob.Type { + rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) { + switch blob.Handle().Type { case restic.DataBlob: - dataPackID = blob.PackID + dataPackID = blob.PackID() case restic.TreeBlob: - treePackID = blob.PackID + treePackID = blob.PackID() } })) return nil diff --git a/cmd/restic/cmd_list_integration_test.go b/cmd/restic/cmd_list_integration_test.go index 655d6da27..9c4c72343 100644 --- a/cmd/restic/cmd_list_integration_test.go +++ b/cmd/restic/cmd_list_integration_test.go @@ -71,8 +71,8 @@ func testListBlobs(t testing.TB, gopts global.Options) (blobSetFromIndex restic. // get blobs from index blobSetFromIndex = restic.NewIDSet() - rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - blobSetFromIndex.Insert(blob.ID) + rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) { + blobSetFromIndex.Insert(blob.Handle().ID) })) return nil }) diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index bbb71972f..eb184e88c 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -75,9 +75,10 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err // tree. If it is not referenced, we have a root tree. trees := make(map[restic.ID]bool) - err = repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - if blob.Type == restic.TreeBlob { - trees[blob.Blob.ID] = false + err = repo.ListBlobs(ctx, func(blob restic.PackBlob) { + h := blob.Handle() + if h.Type == restic.TreeBlob { + trees[h.ID] = false } }) if err != nil { diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index fdf33f074..2e0883a8a 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -483,8 +483,8 @@ func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBl hist[i] = newSizeHistogram(2 * chunker.MaxSize) } - err := repo.ListBlobs(ctx, func(pb restic.PackedBlob) { - hist[pb.Type].Add(uint64(pb.Length)) + err := repo.ListBlobs(ctx, func(pb restic.PackBlob) { + hist[pb.Handle().Type].Add(uint64(pb.CiphertextLength())) }) return hist, err diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index b9a2db4c5..44acf0345 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -270,9 +270,9 @@ func listTreePacks(gopts global.Options, t *testing.T) restic.IDSet { rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks = restic.NewIDSet() - return r.ListBlobs(ctx, func(pb restic.PackedBlob) { - if pb.Type == restic.TreeBlob { - treePacks.Insert(pb.PackID) + return r.ListBlobs(ctx, func(pb restic.PackBlob) { + if pb.Handle().Type == restic.TreeBlob { + treePacks.Insert(pb.PackID()) } }) }) @@ -319,9 +319,9 @@ func removePacksExcept(gopts global.Options, t testing.TB, keep restic.IDSet, re rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks := restic.NewIDSet() - rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) { - if pb.Type == restic.TreeBlob { - treePacks.Insert(pb.PackID) + rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackBlob) { + if pb.Handle().Type == restic.TreeBlob { + treePacks.Insert(pb.PackID()) } })) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 522729a16..2e9941dd6 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -277,8 +277,8 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er ctx, cancel := context.WithCancel(ctx) defer cancel() - err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} + err = c.repo.ListBlobs(ctx, func(blob restic.PackBlob) { + h := blob.Handle() if !c.blobRefs.M.Has(h) { debug.Log("blob %v not referenced", h) blobs = append(blobs, h) diff --git a/internal/repository/checker.go b/internal/repository/checker.go index 845048b4a..3a6986192 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -87,16 +87,18 @@ func NewChecker(repo *Repository) *Checker { } func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) { packs := make(map[restic.ID]restic.BlobType) - err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) { - tpe, exists := packs[pb.PackID] + err := idx.ListBlobs(ctx, func(pb restic.PackBlob) { + packID := pb.PackID() + h := pb.Handle() + tpe, exists := packs[packID] if exists { - if pb.Type != tpe { + if h.Type != tpe { tpe = restic.InvalidBlob } } else { - tpe = pb.Type + tpe = h.Type } - packs[pb.PackID] = tpe + packs[packID] = tpe }) return packs, err } diff --git a/internal/repository/index_list_test.go b/internal/repository/index_list_test.go index 2a9052879..9b7555709 100644 --- a/internal/repository/index_list_test.go +++ b/internal/repository/index_list_test.go @@ -26,8 +26,8 @@ func TestAllIndexBlobs(t *testing.T) { rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) fromMaster := restic.NewBlobSet() - rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - fromMaster.Insert(pb.BlobHandle) + rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + fromMaster.Insert(pb.Handle()) })) rtest.Equals(t, want, fromMaster) diff --git a/internal/repository/index_testutil_test.go b/internal/repository/index_testutil_test.go new file mode 100644 index 000000000..cc44774c4 --- /dev/null +++ b/internal/repository/index_testutil_test.go @@ -0,0 +1,17 @@ +package repository + +import ( + "github.com/restic/restic/internal/restic" +) + +// BlobsInPack returns index entries for blobs stored in packID, sorted by offset. +func BlobsInPack(repo *Repository, packID restic.ID) restic.Blobs { + var blobs restic.Blobs + for pb := range repo.idx.Values() { + if pb.PackID.Equal(packID) { + blobs = append(blobs, pb.Blob) + } + } + blobs.Sort() + return blobs +} diff --git a/internal/repository/pack/pack.go b/internal/repository/pack/pack.go index c065f4f65..25645bddd 100644 --- a/internal/repository/pack/pack.go +++ b/internal/repository/pack/pack.go @@ -65,7 +65,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL p.bytes += uint(n) p.blobs = append(p.blobs, c) - return n + CalculateEntrySize(c), nil + return n + CalculateEntrySize(c.IsCompressed()), nil } var entrySize = uint(binary.Size(restic.BlobType(0)) + 2*headerLengthSize + len(restic.ID{})) @@ -420,8 +420,8 @@ func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) { return b, size, nil } -func CalculateEntrySize(blob restic.Blob) int { - if blob.UncompressedLength != 0 { +func CalculateEntrySize(compressed bool) int { + if compressed { return int(entrySize) } return int(plainEntrySize) @@ -430,7 +430,7 @@ func CalculateEntrySize(blob restic.Blob) int { func CalculateHeaderSize(blobs restic.Blobs) int { size := headerSize for _, blob := range blobs { - size += CalculateEntrySize(blob) + size += CalculateEntrySize(blob.IsCompressed()) } return size } @@ -442,15 +442,16 @@ func CalculateHeaderSize(blobs restic.Blobs) int { func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { packSize := make(map[restic.ID]int64) - err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) { - size, ok := packSize[blob.PackID] + err := mi.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + size, ok := packSize[packID] if !ok { size = headerSize } if !onlyHdr { - size += int64(blob.Length) + size += int64(blob.CiphertextLength()) } - packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob)) + packSize[packID] = size + int64(CalculateEntrySize(blob.IsCompressed())) }) return packSize, err diff --git a/internal/repository/prune.go b/internal/repository/prune.go index 393c2e45b..4890b6a34 100644 --- a/internal/repository/prune.go +++ b/internal/repository/prune.go @@ -132,11 +132,12 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed if len(plan.repackPacks) != 0 { // when repacking, we do not want to keep blobs which are // already contained in kept packs, so delete them from keepBlobs - err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) { - if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) { + err := repo.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + if plan.removePacks.Has(packID) || plan.repackPacks.Has(packID) { return } - keepBlobs.Delete(blob.BlobHandle) + keepBlobs.Delete(blob.Handle()) }) if err != nil { return nil, err @@ -158,8 +159,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i // iterate over all blobs in index to find out which blobs are duplicates // The counter in usedBlobs describes how many instances of the blob exist in the repository index // Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist - err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle + err := idx.ListBlobs(ctx, func(blob restic.PackBlob) { + bh := blob.Handle() count, ok := usedBlobs.Get(bh) if ok { if count < math.MaxUint8 { @@ -208,21 +209,24 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i hasDuplicates := false // iterate over all blobs in index to generate packInfo - err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - ip := indexPack[blob.PackID] + err = idx.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + h := blob.Handle() + + ip := indexPack[packID] // Set blob type if not yet set if ip.tpe == restic.NumBlobTypes { - ip.tpe = blob.Type + ip.tpe = h.Type } // mark mixed packs with "Invalid blob type" - if ip.tpe != blob.Type { + if ip.tpe != h.Type { ip.tpe = restic.InvalidBlob } - bh := blob.BlobHandle - size := uint64(blob.Length) + bh := h + size := uint64(blob.CiphertextLength()) dupCount, _ := usedBlobs.Get(bh) switch { case dupCount >= 2: @@ -252,7 +256,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i ip.uncompressed = true } // update indexPack - indexPack[blob.PackID] = ip + indexPack[packID] = ip }) if err != nil { return nil, nil, err @@ -266,8 +270,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i // - if there are no used blobs in a pack, possibly mark duplicates as "unused" if hasDuplicates { // iterate again over all blobs in index (this is pretty cheap, all in-mem) - err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle + err = idx.ListBlobs(ctx, func(blob restic.PackBlob) { + packID := blob.PackID() + bh := blob.Handle() count, ok := usedBlobs.Get(bh) // skip non-duplicate, aka. normal blobs // count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining @@ -275,8 +280,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i return } - ip := indexPack[blob.PackID] - size := uint64(blob.Length) + ip := indexPack[packID] + size := uint64(blob.CiphertextLength()) switch { case ip.usedBlobs > 0, ip.duplicateBlobs == ip.unusedBlobs, count == 0: // other used blobs in pack, only duplicate blobs or "last" occurrence -> transition to used @@ -304,7 +309,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i usedBlobs.Set(bh, count) } // update indexPack - indexPack[blob.PackID] = ip + indexPack[packID] = ip }) if err != nil { return nil, nil, err diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index a8c532a65..297eaa20d 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -16,8 +16,8 @@ import ( func listBlobs(repo restic.Repository) restic.BlobSet { blobs := restic.NewBlobSet() - _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - blobs.Insert(pb.BlobHandle) + _ = repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + blobs.Insert(pb.Handle()) }) return blobs } @@ -66,11 +66,12 @@ func testRepairBrokenPack(t *testing.T, version uint) { // find blob that starts at offset 0 var damagedBlob restic.BlobHandle - _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - if pb.PackID == damagedID && pb.Offset == 0 { - damagedBlob = pb.BlobHandle + for _, blob := range repository.BlobsInPack(repo, damagedID) { + if blob.Offset == 0 { + damagedBlob = blob.BlobHandle + break } - }) + } return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob) }, @@ -87,11 +88,11 @@ func testRepairBrokenPack(t *testing.T, version uint) { // all blobs in the file are broken damagedBlobs := restic.NewBlobSet() - _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { - if pb.PackID == damagedID { - damagedBlobs.Insert(pb.BlobHandle) + rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) { + if pb.PackID().Equal(damagedID) { + damagedBlobs.Insert(pb.Handle()) } - }) + })) return restic.NewIDSet(damagedID), damagedBlobs }, }, { diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 500e9b06d..4ecdc9859 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -681,12 +681,12 @@ func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bo // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. -func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error { +func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) error { for blob := range r.idx.Values() { if ctx.Err() != nil { return ctx.Err() } - fn(blob) + fn(restic.AsPackBlob(blob)) } return nil } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 7986e2f74..1fe567b01 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -30,7 +30,7 @@ type Repository interface { NewAssociatedBlobSet() AssociatedBlobSet // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. - ListBlobs(ctx context.Context, fn func(PackedBlob)) error + ListBlobs(ctx context.Context, fn func(PackBlob)) error // ListPackHandles returns the blob handles stored in the pack file header. ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error) @@ -152,7 +152,7 @@ type Unpacked[FT FileTypes] interface { } type ListBlobser interface { - ListBlobs(ctx context.Context, fn func(PackedBlob)) error + ListBlobs(ctx context.Context, fn func(PackBlob)) error } type BlobLoader interface {