restic: change ListBlobs to return PackBlob

PackBlob is a limited interface that only exposes a part of the
information provided by PackedBlob. Most of the changes are switches
from direct value lookups to the interface methods, with a few larger
changes to let the tests still work.

cmd/restic/cmd_find.go

@@ -526,22 +526,23 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 
 	// remember which packs were found in the index
 	indexPackIDs := make(map[string]struct{})
-	err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) {
+	err := f.repo.ListBlobs(wctx, func(pb restic.PackBlob) {
-		idStr := pb.PackID.String()
+		packID := pb.PackID()
+		idStr := packID.String()
 		// keep entry in packIDs as Each() returns individual index entries
 		matchingID := false
 		if _, ok := packIDs[idStr]; ok {
 			matchingID = true
 		} else {
-			if _, ok := packIDs[pb.PackID.Str()]; ok {
+			if _, ok := packIDs[packID.Str()]; ok {
 				// expand id
-				delete(packIDs, pb.PackID.Str())
+				delete(packIDs, packID.Str())
 				packIDs[idStr] = struct{}{}
 				matchingID = true
 			}
 		}
 		if matchingID {
-			f.blobIDs[pb.ID.String()] = struct{}{}
+			f.blobIDs[pb.Handle().ID.String()] = struct{}{}
 			indexPackIDs[idStr] = struct{}{}
 		}
 	})

cmd/restic/cmd_find_integration_test.go

@@ -179,9 +179,10 @@ func TestFindPackfile(t *testing.T) {
 
 		packID := restic.ID{}
 		done := false
-		err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) {
+		err = repo.ListBlobs(ctx, func(pb restic.PackBlob) {
-			if !done && pb.Type == restic.TreeBlob {
+			h := pb.Handle()
-				packID = pb.PackID
+			if !done && h.Type == restic.TreeBlob {
+				packID = pb.PackID()
 				done = true
 			}
 		})
@@ -236,12 +237,12 @@ func TestFindPackID(t *testing.T) {
 		// load Index
 		rtest.OK(t, repo.LoadIndex(ctx, nil))
 		// go through all index entries and collect data and tree packfile(s)
-		rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
+		rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) {
-			switch blob.Type {
+			switch blob.Handle().Type {
 			case restic.DataBlob:
-				dataPackID = blob.PackID
+				dataPackID = blob.PackID()
 			case restic.TreeBlob:
-				treePackID = blob.PackID
+				treePackID = blob.PackID()
 			}
 		}))
 		return nil

cmd/restic/cmd_list_integration_test.go

@@ -71,8 +71,8 @@ func testListBlobs(t testing.TB, gopts global.Options) (blobSetFromIndex restic.
 
 		// get blobs from index
 		blobSetFromIndex = restic.NewIDSet()
-		rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
+		rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) {
-			blobSetFromIndex.Insert(blob.ID)
+			blobSetFromIndex.Insert(blob.Handle().ID)
 		}))
 		return nil
 	})

cmd/restic/cmd_recover.go

@@ -75,9 +75,10 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
 	// tree. If it is not referenced, we have a root tree.
 	trees := make(map[restic.ID]bool)
 
-	err = repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
+	err = repo.ListBlobs(ctx, func(blob restic.PackBlob) {
-		if blob.Type == restic.TreeBlob {
+		h := blob.Handle()
-			trees[blob.Blob.ID] = false
+		if h.Type == restic.TreeBlob {
+			trees[h.ID] = false
 		}
 	})
 	if err != nil {

cmd/restic/cmd_stats.go

@@ -414,8 +414,8 @@ func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBl
 		hist[i] = newSizeHistogram(2 * chunker.MaxSize)
 	}
 
-	err := repo.ListBlobs(ctx, func(pb restic.PackedBlob) {
+	err := repo.ListBlobs(ctx, func(pb restic.PackBlob) {
-		hist[pb.Type].Add(uint64(pb.Length))
+		hist[pb.Handle().Type].Add(uint64(pb.CiphertextLength()))
 	})
 
 	return hist, err

cmd/restic/integration_helpers_test.go

@@ -269,9 +269,9 @@ func listTreePacks(gopts global.Options, t *testing.T) restic.IDSet {
 
 		rtest.OK(t, r.LoadIndex(ctx, nil))
 		treePacks = restic.NewIDSet()
-		return r.ListBlobs(ctx, func(pb restic.PackedBlob) {
+		return r.ListBlobs(ctx, func(pb restic.PackBlob) {
-			if pb.Type == restic.TreeBlob {
+			if pb.Handle().Type == restic.TreeBlob {
-				treePacks.Insert(pb.PackID)
+				treePacks.Insert(pb.PackID())
 			}
 		})
 	})
@@ -318,9 +318,9 @@ func removePacksExcept(gopts global.Options, t testing.TB, keep restic.IDSet, re
 		rtest.OK(t, r.LoadIndex(ctx, nil))
 
 		treePacks := restic.NewIDSet()
-		rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) {
+		rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackBlob) {
-			if pb.Type == restic.TreeBlob {
+			if pb.Handle().Type == restic.TreeBlob {
-				treePacks.Insert(pb.PackID)
+				treePacks.Insert(pb.PackID())
 			}
 		}))
 

internal/checker/checker.go

@@ -277,8 +277,8 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
+	err = c.repo.ListBlobs(ctx, func(blob restic.PackBlob) {
-		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
+		h := blob.Handle()
 		if !c.blobRefs.M.Has(h) {
 			debug.Log("blob %v not referenced", h)
 			blobs = append(blobs, h)

internal/repository/checker.go

@@ -87,16 +87,18 @@ func NewChecker(repo *Repository) *Checker {
 }
 func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) {
 	packs := make(map[restic.ID]restic.BlobType)
-	err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) {
+	err := idx.ListBlobs(ctx, func(pb restic.PackBlob) {
-		tpe, exists := packs[pb.PackID]
+		packID := pb.PackID()
+		h := pb.Handle()
+		tpe, exists := packs[packID]
 		if exists {
-			if pb.Type != tpe {
+			if h.Type != tpe {
 				tpe = restic.InvalidBlob
 			}
 		} else {
-			tpe = pb.Type
+			tpe = h.Type
 		}
-		packs[pb.PackID] = tpe
+		packs[packID] = tpe
 	})
 	return packs, err
 }

internal/repository/index_list_test.go

@@ -26,8 +26,8 @@ func TestAllIndexBlobs(t *testing.T) {
 	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
 
 	fromMaster := restic.NewBlobSet()
-	rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
+	rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
-		fromMaster.Insert(pb.BlobHandle)
+		fromMaster.Insert(pb.Handle())
 	}))
 	rtest.Equals(t, want, fromMaster)
 

internal/repository/index_testutil_test.go

@@ -0,0 +1,17 @@
+package repository
+
+import (
+	"github.com/restic/restic/internal/restic"
+)
+
+// BlobsInPack returns index entries for blobs stored in packID, sorted by offset.
+func BlobsInPack(repo *Repository, packID restic.ID) restic.Blobs {
+	var blobs restic.Blobs
+	for pb := range repo.idx.Values() {
+		if pb.PackID.Equal(packID) {
+			blobs = append(blobs, pb.Blob)
+		}
+	}
+	blobs.Sort()
+	return blobs
+}

internal/repository/pack/pack.go

@@ -65,7 +65,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL
 	p.bytes += uint(n)
 	p.blobs = append(p.blobs, c)
 
-	return n + CalculateEntrySize(c), nil
+	return n + CalculateEntrySize(c.IsCompressed()), nil
 }
 
 var entrySize = uint(binary.Size(restic.BlobType(0)) + 2*headerLengthSize + len(restic.ID{}))
@@ -420,8 +420,8 @@ func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) {
 	return b, size, nil
 }
 
-func CalculateEntrySize(blob restic.Blob) int {
+func CalculateEntrySize(compressed bool) int {
-	if blob.UncompressedLength != 0 {
+	if compressed {
 		return int(entrySize)
 	}
 	return int(plainEntrySize)
@@ -430,7 +430,7 @@ func CalculateEntrySize(blob restic.Blob) int {
 func CalculateHeaderSize(blobs restic.Blobs) int {
 	size := headerSize
 	for _, blob := range blobs {
-		size += CalculateEntrySize(blob)
+		size += CalculateEntrySize(blob.IsCompressed())
 	}
 	return size
 }
@@ -442,15 +442,16 @@ func CalculateHeaderSize(blobs restic.Blobs) int {
 func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) {
 	packSize := make(map[restic.ID]int64)
 
-	err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) {
+	err := mi.ListBlobs(ctx, func(blob restic.PackBlob) {
-		size, ok := packSize[blob.PackID]
+		packID := blob.PackID()
+		size, ok := packSize[packID]
 		if !ok {
 			size = headerSize
 		}
 		if !onlyHdr {
-			size += int64(blob.Length)
+			size += int64(blob.CiphertextLength())
 		}
-		packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob))
+		packSize[packID] = size + int64(CalculateEntrySize(blob.IsCompressed()))
 	})
 
 	return packSize, err

internal/repository/prune.go

@@ -142,11 +142,12 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed
 	if len(plan.repackPacks) != 0 {
 		// when repacking, we do not want to keep blobs which are
 		// already contained in kept packs, so delete them from keepBlobs
-		err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
+		err := repo.ListBlobs(ctx, func(blob restic.PackBlob) {
-			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
+			packID := blob.PackID()
+			if plan.removePacks.Has(packID) || plan.repackPacks.Has(packID) {
 				return
 			}
-			keepBlobs.Delete(blob.BlobHandle)
+			keepBlobs.Delete(blob.Handle())
 		})
 		if err != nil {
 			return nil, err
@@ -179,8 +180,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 	// iterate over all blobs in index to find out which blobs are duplicates
 	// The counter in usedBlobs describes how many instances of the blob exist in the repository index
 	// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
-	err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
+	err := idx.ListBlobs(ctx, func(blob restic.PackBlob) {
-		bh := blob.BlobHandle
+		bh := blob.Handle()
 		count, ok := usedBlobs.Get(bh)
 		if ok {
 			if count < math.MaxUint8 {
@@ -229,21 +230,24 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 
 	hasDuplicates := false
 	// iterate over all blobs in index to generate packInfo
-	err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
+	err = idx.ListBlobs(ctx, func(blob restic.PackBlob) {
-		ip := indexPack[blob.PackID]
+		packID := blob.PackID()
+		h := blob.Handle()
+
+		ip := indexPack[packID]
 
 		// Set blob type if not yet set
 		if ip.tpe == restic.NumBlobTypes {
-			ip.tpe = blob.Type
+			ip.tpe = h.Type
 		}
 
 		// mark mixed packs with "Invalid blob type"
-		if ip.tpe != blob.Type {
+		if ip.tpe != h.Type {
 			ip.tpe = restic.InvalidBlob
 		}
 
-		bh := blob.BlobHandle
+		bh := h
-		size := uint64(blob.Length)
+		size := uint64(blob.CiphertextLength())
 		dupCount, _ := usedBlobs.Get(bh)
 		switch {
 		case dupCount >= 2:
@@ -273,7 +277,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 			ip.uncompressed = true
 		}
 		// update indexPack
-		indexPack[blob.PackID] = ip
+		indexPack[packID] = ip
 	})
 	if err != nil {
 		return nil, nil, err
@@ -287,8 +291,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
 	if hasDuplicates {
 		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
-		err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
+		err = idx.ListBlobs(ctx, func(blob restic.PackBlob) {
-			bh := blob.BlobHandle
+			packID := blob.PackID()
+			bh := blob.Handle()
 			count, ok := usedBlobs.Get(bh)
 			// skip non-duplicate, aka. normal blobs
 			// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
@@ -296,8 +301,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 				return
 			}
 
-			ip := indexPack[blob.PackID]
+			ip := indexPack[packID]
-			size := uint64(blob.Length)
+			size := uint64(blob.CiphertextLength())
 			switch {
 			case ip.usedBlobs > 0, ip.duplicateBlobs == ip.unusedBlobs, count == 0:
 				// other used blobs in pack, only duplicate blobs or "last" occurrence ->  transition to used
@@ -325,7 +330,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 				usedBlobs.Set(bh, count)
 			}
 			// update indexPack
-			indexPack[blob.PackID] = ip
+			indexPack[packID] = ip
 		})
 		if err != nil {
 			return nil, nil, err

internal/repository/repair_pack_test.go

@@ -16,8 +16,8 @@ import (
 
 func listBlobs(repo restic.Repository) restic.BlobSet {
 	blobs := restic.NewBlobSet()
-	_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
+	_ = repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
-		blobs.Insert(pb.BlobHandle)
+		blobs.Insert(pb.Handle())
 	})
 	return blobs
 }
@@ -66,11 +66,12 @@ func testRepairBrokenPack(t *testing.T, version uint) {
 
 				// find blob that starts at offset 0
 				var damagedBlob restic.BlobHandle
-				_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
+				for _, blob := range repository.BlobsInPack(repo, damagedID) {
-					if pb.PackID == damagedID && pb.Offset == 0 {
+					if blob.Offset == 0 {
-						damagedBlob = pb.BlobHandle
+						damagedBlob = blob.BlobHandle
+						break
 					}
-				})
+				}
 
 				return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob)
 			},
@@ -87,11 +88,11 @@ func testRepairBrokenPack(t *testing.T, version uint) {
 
 				// all blobs in the file are broken
 				damagedBlobs := restic.NewBlobSet()
-				_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
+				rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
-					if pb.PackID == damagedID {
+					if pb.PackID().Equal(damagedID) {
-						damagedBlobs.Insert(pb.BlobHandle)
+						damagedBlobs.Insert(pb.Handle())
 					}
-				})
+				}))
 				return restic.NewIDSet(damagedID), damagedBlobs
 			},
 		}, {

internal/repository/repository.go

@@ -683,12 +683,12 @@ func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bo
 
 // ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
 // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
-func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error {
+func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) error {
 	for blob := range r.idx.Values() {
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
-		fn(blob)
+		fn(restic.AsPackBlob(blob))
 	}
 	return nil
 }

internal/restic/repository.go

@@ -30,7 +30,7 @@ type Repository interface {
 	NewAssociatedBlobSet() AssociatedBlobSet
 	// ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
 	// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
-	ListBlobs(ctx context.Context, fn func(PackedBlob)) error
+	ListBlobs(ctx context.Context, fn func(PackBlob)) error
 	// ListPackHandles returns the blob handles stored in the pack file header.
 	ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error)
 
@@ -152,7 +152,7 @@ type Unpacked[FT FileTypes] interface {
 }
 
 type ListBlobser interface {
-	ListBlobs(ctx context.Context, fn func(PackedBlob)) error
+	ListBlobs(ctx context.Context, fn func(PackBlob)) error
 }
 
 type BlobLoader interface {