restic: change ListBlobs to return PackBlob

PackBlob is a limited interface that only exposes a part of the
information provided by PackedBlob. Most of the changes are switches
from direct value lookups to the interface methods, with a few larger
changes to let the tests still work.

internal/checker/checker.go

@@ -277,8 +277,8 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
-		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
+	err = c.repo.ListBlobs(ctx, func(blob restic.PackBlob) {
+		h := blob.Handle()
 		if !c.blobRefs.M.Has(h) {
 			debug.Log("blob %v not referenced", h)
 			blobs = append(blobs, h)

internal/repository/checker.go

@@ -87,16 +87,18 @@ func NewChecker(repo *Repository) *Checker {
 }
 func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) {
 	packs := make(map[restic.ID]restic.BlobType)
-	err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) {
-		tpe, exists := packs[pb.PackID]
+	err := idx.ListBlobs(ctx, func(pb restic.PackBlob) {
+		packID := pb.PackID()
+		h := pb.Handle()
+		tpe, exists := packs[packID]
 		if exists {
-			if pb.Type != tpe {
+			if h.Type != tpe {
 				tpe = restic.InvalidBlob
 			}
 		} else {
-			tpe = pb.Type
+			tpe = h.Type
 		}
-		packs[pb.PackID] = tpe
+		packs[packID] = tpe
 	})
 	return packs, err
 }

internal/repository/index_list_test.go

@@ -26,8 +26,8 @@ func TestAllIndexBlobs(t *testing.T) {
 	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
 
 	fromMaster := restic.NewBlobSet()
-	rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
-		fromMaster.Insert(pb.BlobHandle)
+	rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
+		fromMaster.Insert(pb.Handle())
 	}))
 	rtest.Equals(t, want, fromMaster)
 

internal/repository/index_testutil_test.go

@@ -0,0 +1,17 @@
+package repository
+
+import (
+	"github.com/restic/restic/internal/restic"
+)
+
+// BlobsInPack returns index entries for blobs stored in packID, sorted by offset.
+func BlobsInPack(repo *Repository, packID restic.ID) restic.Blobs {
+	var blobs restic.Blobs
+	for pb := range repo.idx.Values() {
+		if pb.PackID.Equal(packID) {
+			blobs = append(blobs, pb.Blob)
+		}
+	}
+	blobs.Sort()
+	return blobs
+}

internal/repository/pack/pack.go

@@ -65,7 +65,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL
 	p.bytes += uint(n)
 	p.blobs = append(p.blobs, c)
 
-	return n + CalculateEntrySize(c), nil
+	return n + CalculateEntrySize(c.IsCompressed()), nil
 }
 
 var entrySize = uint(binary.Size(restic.BlobType(0)) + 2*headerLengthSize + len(restic.ID{}))
@@ -420,8 +420,8 @@ func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) {
 	return b, size, nil
 }
 
-func CalculateEntrySize(blob restic.Blob) int {
-	if blob.UncompressedLength != 0 {
+func CalculateEntrySize(compressed bool) int {
+	if compressed {
 		return int(entrySize)
 	}
 	return int(plainEntrySize)
@@ -430,7 +430,7 @@ func CalculateEntrySize(blob restic.Blob) int {
 func CalculateHeaderSize(blobs restic.Blobs) int {
 	size := headerSize
 	for _, blob := range blobs {
-		size += CalculateEntrySize(blob)
+		size += CalculateEntrySize(blob.IsCompressed())
 	}
 	return size
 }
@@ -442,15 +442,16 @@ func CalculateHeaderSize(blobs restic.Blobs) int {
 func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) {
 	packSize := make(map[restic.ID]int64)
 
-	err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) {
-		size, ok := packSize[blob.PackID]
+	err := mi.ListBlobs(ctx, func(blob restic.PackBlob) {
+		packID := blob.PackID()
+		size, ok := packSize[packID]
 		if !ok {
 			size = headerSize
 		}
 		if !onlyHdr {
-			size += int64(blob.Length)
+			size += int64(blob.CiphertextLength())
 		}
-		packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob))
+		packSize[packID] = size + int64(CalculateEntrySize(blob.IsCompressed()))
 	})
 
 	return packSize, err

internal/repository/prune.go

@@ -142,11 +142,12 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed
 	if len(plan.repackPacks) != 0 {
 		// when repacking, we do not want to keep blobs which are
 		// already contained in kept packs, so delete them from keepBlobs
-		err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
-			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
+		err := repo.ListBlobs(ctx, func(blob restic.PackBlob) {
+			packID := blob.PackID()
+			if plan.removePacks.Has(packID) || plan.repackPacks.Has(packID) {
 				return
 			}
-			keepBlobs.Delete(blob.BlobHandle)
+			keepBlobs.Delete(blob.Handle())
 		})
 		if err != nil {
 			return nil, err
@@ -179,8 +180,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 	// iterate over all blobs in index to find out which blobs are duplicates
 	// The counter in usedBlobs describes how many instances of the blob exist in the repository index
 	// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
-	err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
-		bh := blob.BlobHandle
+	err := idx.ListBlobs(ctx, func(blob restic.PackBlob) {
+		bh := blob.Handle()
 		count, ok := usedBlobs.Get(bh)
 		if ok {
 			if count < math.MaxUint8 {
@@ -229,21 +230,24 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 
 	hasDuplicates := false
 	// iterate over all blobs in index to generate packInfo
-	err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
-		ip := indexPack[blob.PackID]
+	err = idx.ListBlobs(ctx, func(blob restic.PackBlob) {
+		packID := blob.PackID()
+		h := blob.Handle()
+
+		ip := indexPack[packID]
 
 		// Set blob type if not yet set
 		if ip.tpe == restic.NumBlobTypes {
-			ip.tpe = blob.Type
+			ip.tpe = h.Type
 		}
 
 		// mark mixed packs with "Invalid blob type"
-		if ip.tpe != blob.Type {
+		if ip.tpe != h.Type {
 			ip.tpe = restic.InvalidBlob
 		}
 
-		bh := blob.BlobHandle
-		size := uint64(blob.Length)
+		bh := h
+		size := uint64(blob.CiphertextLength())
 		dupCount, _ := usedBlobs.Get(bh)
 		switch {
 		case dupCount >= 2:
@@ -273,7 +277,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 			ip.uncompressed = true
 		}
 		// update indexPack
-		indexPack[blob.PackID] = ip
+		indexPack[packID] = ip
 	})
 	if err != nil {
 		return nil, nil, err
@@ -287,8 +291,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
 	if hasDuplicates {
 		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
-		err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
-			bh := blob.BlobHandle
+		err = idx.ListBlobs(ctx, func(blob restic.PackBlob) {
+			packID := blob.PackID()
+			bh := blob.Handle()
 			count, ok := usedBlobs.Get(bh)
 			// skip non-duplicate, aka. normal blobs
 			// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
@@ -296,8 +301,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 				return
 			}
 
-			ip := indexPack[blob.PackID]
-			size := uint64(blob.Length)
+			ip := indexPack[packID]
+			size := uint64(blob.CiphertextLength())
 			switch {
 			case ip.usedBlobs > 0, ip.duplicateBlobs == ip.unusedBlobs, count == 0:
 				// other used blobs in pack, only duplicate blobs or "last" occurrence ->  transition to used
@@ -325,7 +330,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
 				usedBlobs.Set(bh, count)
 			}
 			// update indexPack
-			indexPack[blob.PackID] = ip
+			indexPack[packID] = ip
 		})
 		if err != nil {
 			return nil, nil, err

internal/repository/repair_pack_test.go

@@ -16,8 +16,8 @@ import (
 
 func listBlobs(repo restic.Repository) restic.BlobSet {
 	blobs := restic.NewBlobSet()
-	_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
-		blobs.Insert(pb.BlobHandle)
+	_ = repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
+		blobs.Insert(pb.Handle())
 	})
 	return blobs
 }
@@ -66,11 +66,12 @@ func testRepairBrokenPack(t *testing.T, version uint) {
 
 				// find blob that starts at offset 0
 				var damagedBlob restic.BlobHandle
-				_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
-					if pb.PackID == damagedID && pb.Offset == 0 {
-						damagedBlob = pb.BlobHandle
+				for _, blob := range repository.BlobsInPack(repo, damagedID) {
+					if blob.Offset == 0 {
+						damagedBlob = blob.BlobHandle
+						break
 					}
-				})
+				}
 
 				return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob)
 			},
@@ -87,11 +88,11 @@ func testRepairBrokenPack(t *testing.T, version uint) {
 
 				// all blobs in the file are broken
 				damagedBlobs := restic.NewBlobSet()
-				_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
-					if pb.PackID == damagedID {
-						damagedBlobs.Insert(pb.BlobHandle)
+				rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
+					if pb.PackID().Equal(damagedID) {
+						damagedBlobs.Insert(pb.Handle())
 					}
-				})
+				}))
 				return restic.NewIDSet(damagedID), damagedBlobs
 			},
 		}, {

internal/repository/repository.go

@@ -683,12 +683,12 @@ func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bo
 
 // ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
 // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
-func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error {
+func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) error {
 	for blob := range r.idx.Values() {
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
-		fn(blob)
+		fn(restic.AsPackBlob(blob))
 	}
 	return nil
 }

internal/restic/repository.go

@@ -30,7 +30,7 @@ type Repository interface {
 	NewAssociatedBlobSet() AssociatedBlobSet
 	// ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
 	// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
-	ListBlobs(ctx context.Context, fn func(PackedBlob)) error
+	ListBlobs(ctx context.Context, fn func(PackBlob)) error
 	// ListPackHandles returns the blob handles stored in the pack file header.
 	ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error)
 
@@ -152,7 +152,7 @@ type Unpacked[FT FileTypes] interface {
 }
 
 type ListBlobser interface {
-	ListBlobs(ctx context.Context, fn func(PackedBlob)) error
+	ListBlobs(ctx context.Context, fn func(PackBlob)) error
 }
 
 type BlobLoader interface {