restic: change LookupBlob to return []PackBlob

cmd/restic/cmd_copy.go

@@ -268,7 +268,7 @@ func copyTree(ctx context.Context, srcRepo *repository.Repository, dstRepo resti
 			pb := srcRepo.LookupBlob(h.Type, h.ID)
 			copyBlobs.Insert(h)
 			for _, p := range pb {
-				packList.Insert(p.PackID)
+				packList.Insert(p.PackID())
 			}
 		}
 	}
@@ -317,9 +317,9 @@ func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, pa
 	countBlobs := 0
 	sizeBlobs := uint64(0)
 	for blob := range copyBlobs.Keys() {
-		for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) {
+		for _, pb := range srcRepo.LookupBlob(blob.Type, blob.ID) {
 			countBlobs++
-			sizeBlobs += uint64(blob.Length)
+			sizeBlobs += uint64(pb.CiphertextLength())
 			break
 		}
 	}

cmd/restic/cmd_find.go

@@ -577,9 +577,9 @@ func (f *Finder) findObjectPack(id string, t restic.BlobType) {
 	}
 
 	for _, b := range blobs {
-		if b.ID.Equal(rid) {
-			f.printer.S("Object belongs to pack %s", b.PackID)
-			f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String())
+		if b.Handle().ID.Equal(rid) {
+			f.printer.S("Object belongs to pack %s", b.PackID())
+			f.printer.S(" ... Pack %s: %v", b.PackID().String(), b.Handle())
 			break
 		}
 	}

cmd/restic/cmd_stats.go

@@ -166,16 +166,16 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
 			if len(pbs) == 0 {
 				return fmt.Errorf("blob %v not found", blobHandle)
 			}
-			stats.TotalSize += uint64(pbs[0].Length)
+			stats.TotalSize += uint64(pbs[0].CiphertextLength())
 			if repo.Config().Version >= 2 {
-				stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
+				stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength())))
 				if pbs[0].IsCompressed() {
-					stats.TotalCompressedBlobsSize += uint64(pbs[0].Length)
-					stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
+					stats.TotalCompressedBlobsSize += uint64(pbs[0].CiphertextLength())
+					stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength())))
 				}
 			}
 			stats.TotalBlobCount++
-			statsProgress.update(0, 1, uint64(pbs[0].Length))
+			statsProgress.update(0, 1, uint64(pbs[0].CiphertextLength()))
 		}
 		if stats.TotalCompressedBlobsSize > 0 {
 			stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize)

internal/checker/checker.go

@@ -308,7 +308,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
 		// convert used blobs into their encompassing packfiles
 		for bh := range c.blobRefs.M.Keys() {
 			for _, pb := range c.repo.LookupBlob(bh.Type, bh.ID) {
-				filteredPacks[pb.PackID] = allPacks[pb.PackID]
+				filteredPacks[pb.PackID()] = allPacks[pb.PackID()]
 			}
 		}
 

internal/repository/checker.go

@@ -464,8 +464,8 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest
 	for _, blob := range blobs {
 		// Check if blob is contained in index and position is correct
 		idxHas := false
-		for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
-			if pb.PackID == id && pb.Blob == blob {
+		for _, pb := range r.idx.Lookup(blob.BlobHandle) {
+			if pb.PackID.Equal(id) && pb.Blob == blob {
 				idxHas = true
 				break
 			}

internal/repository/repack_test.go

@@ -141,7 +141,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
 		}
 
 		for _, pb := range list {
-			packs.Insert(pb.PackID)
+			packs.Insert(pb.PackID())
 		}
 	}
 
@@ -221,8 +221,8 @@ func testRepack(t *testing.T, version uint) {
 
 		pb := list[0]
 
-		if removePacks.Has(pb.PackID) {
-			t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID)
+		if removePacks.Has(pb.PackID()) {
+			t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID())
 		}
 	}
 

internal/repository/repository.go

@@ -670,8 +670,13 @@ func (r *Repository) Connections() uint {
 	return r.be.Properties().Connections
 }
 
-func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackedBlob {
-	return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id})
+func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob {
+	entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id})
+	out := make([]restic.PackBlob, len(entries))
+	for i, pb := range entries {
+		out[i] = restic.AsPackBlob(pb)
+	}
+	return out
 }
 
 // LookupBlobSize returns the size of blob id. Also returns pending blobs.

internal/repository/repository_test.go

@@ -211,7 +211,7 @@ func TestLoadBlobBroken(t *testing.T) {
 	data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil)
 	rtest.OK(t, err)
 	rtest.Assert(t, bytes.Equal(buf, data), "data mismatch")
-	pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID
+	pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID()
 	rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached")
 }
 
@@ -400,11 +400,12 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
 		rtest.OK(t, err)
 
 		for pb := range idx.Values() {
-			if _, ok := packEntries[pb.PackID]; !ok {
-				packEntries[pb.PackID] = make(map[restic.ID]struct{})
+			packID := pb.PackID
+			if _, ok := packEntries[packID]; !ok {
+				packEntries[packID] = make(map[restic.ID]struct{})
 			}
 
-			packEntries[pb.PackID][id] = struct{}{}
+			packEntries[packID][id] = struct{}{}
 		}
 		return nil
 	})
@@ -445,7 +446,7 @@ func TestListPack(t *testing.T) {
 	repo.UseCache(c, t.Logf)
 
 	// Forcibly cache pack file
-	packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID
+	packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID()
 	rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil }))
 
 	// Get size to list pack

internal/restic/repository.go

@@ -24,7 +24,7 @@ type Repository interface {
 
 	LoadIndex(ctx context.Context, p TerminalCounterFactory) error
 
-	LookupBlob(t BlobType, id ID) []PackedBlob
+	LookupBlob(t BlobType, id ID) []PackBlob
 	LookupBlobSize(t BlobType, id ID) (size uint, exists bool)
 
 	NewAssociatedBlobSet() AssociatedBlobSet

internal/restorer/filerestorer.go

@@ -47,7 +47,7 @@ type startWarmupFn func(context.Context, restic.IDSet) (restic.WarmupJob, error)
 
 // fileRestorer restores set of files
 type fileRestorer struct {
-	idx         func(restic.BlobType, restic.ID) []restic.PackedBlob
+	idx         func(restic.BlobType, restic.ID) []restic.PackBlob
 	blobsLoader blobsLoaderFn
 
 	startWarmup startWarmupFn
@@ -68,7 +68,7 @@ type fileRestorer struct {
 
 func newFileRestorer(dst string,
 	blobsLoader blobsLoaderFn,
-	idx func(restic.BlobType, restic.ID) []restic.PackedBlob,
+	idx func(restic.BlobType, restic.ID) []restic.PackBlob,
 	connections uint,
 	sparse bool,
 	allowRecursiveDelete bool,
@@ -102,7 +102,7 @@ func (r *fileRestorer) targetPath(location string) string {
 	return filepath.Join(r.dst, location)
 }
 
-func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int, fileOffset int64)) error {
+func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(blob restic.PackBlob, idx int, fileOffset int64)) error {
 	if len(blobIDs) == 0 {
 		return nil
 	}
@@ -114,8 +114,8 @@ func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID
 			return errors.Errorf("Unknown blob %s", blobID.String())
 		}
 		pb := packs[0]
-		fn(pb.PackID, pb.Blob, i, fileOffset)
-		fileOffset += int64(pb.DataLength())
+		fn(pb, i, fileOffset)
+		fileOffset += int64(pb.PlaintextLength())
 	}
 
 	return nil
@@ -143,14 +143,15 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 			file.blobs = packsMap
 		}
 		restoredBlobs := false
-		err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) {
+		err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) {
+			packID := blob.PackID()
 			if !file.state.HasMatchingBlob(idx) {
 				if largeFile {
-					packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset})
+					packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.Handle().ID, offset: fileOffset})
 				}
 				restoredBlobs = true
 			} else {
-				r.reportBlobProgress(file, uint64(blob.DataLength()))
+				r.reportBlobProgress(file, uint64(blob.PlaintextLength()))
 				// completely ignore blob
 				return
 			}
@@ -164,7 +165,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 				packOrder = append(packOrder, packID)
 			}
 			pack.files[file] = struct{}{}
-			if blob.ID.Equal(r.zeroChunk) {
+			if blob.Handle().ID.Equal(r.zeroChunk) {
 				file.sparse = r.sparse
 			}
 		})
@@ -278,9 +279,9 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
 			blobInfo.files[file] = append(blobInfo.files[file], fileOffset)
 		}
 		if fileBlobs, ok := file.blobs.(restic.IDs); ok {
-			err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) {
-				if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) {
-					addBlob(blob.BlobHandle, fileOffset)
+			err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) {
+				if blob.PackID().Equal(pack.id) && !file.state.HasMatchingBlob(idx) {
+					addBlob(blob.Handle(), fileOffset)
 				}
 			})
 			if err != nil {
@@ -291,8 +292,8 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
 			for _, blob := range packsMap[pack.id] {
 				idxPacks := r.idx(restic.DataBlob, blob.id)
 				for _, idxPack := range idxPacks {
-					if idxPack.PackID.Equal(pack.id) {
-						addBlob(idxPack.BlobHandle, blob.offset)
+					if idxPack.PackID().Equal(pack.id) {
+						addBlob(idxPack.Handle(), blob.offset)
 						break
 					}
 				}

internal/restorer/filerestorer_test.go

@@ -30,11 +30,32 @@ type TestWarmupJob struct {
 	waitCalled   bool
 }
 
+type testPackBlob struct {
+	packID     restic.ID
+	handle     restic.BlobHandle
+	offset     uint
+	ciphertext uint
+	plaintext  uint
+	compressed bool
+}
+
+var _ restic.PackBlob = (*testPackBlob)(nil)
+
+func (pb *testPackBlob) PackID() restic.ID { return pb.packID }
+
+func (pb *testPackBlob) Handle() restic.BlobHandle { return pb.handle }
+
+func (pb *testPackBlob) CiphertextLength() uint { return pb.ciphertext }
+
+func (pb *testPackBlob) PlaintextLength() uint { return pb.plaintext }
+
+func (pb *testPackBlob) IsCompressed() bool { return pb.compressed }
+
 type TestRepo struct {
 	packsIDToData map[restic.ID][]byte
 
 	// blobs and files
-	blobs              map[restic.ID][]restic.PackedBlob
+	blobs              map[restic.ID][]restic.PackBlob
 	files              []*fileInfo
 	filesPathToContent map[string]string
 
@@ -44,7 +65,7 @@ type TestRepo struct {
 	loader blobsLoaderFn
 }
 
-func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackedBlob {
+func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackBlob {
 	packs := i.blobs[id]
 	return packs
 }
@@ -69,10 +90,16 @@ func (job *TestWarmupJob) Wait(_ context.Context) error {
 }
 
 func newTestRepo(content []TestFile) *TestRepo {
+	type packBlobLayout struct {
+		offset     uint
+		ciphertext uint
+		plaintext  uint
+		compressed bool
+	}
 	type Pack struct {
 		name  string
 		data  []byte
-		blobs map[restic.ID]restic.Blob
+		blobs map[restic.ID]packBlobLayout
 	}
 	packs := make(map[string]Pack)
 	filesPathToContent := make(map[string]string)
@@ -86,21 +113,19 @@ func newTestRepo(content []TestFile) *TestRepo {
 			var pack Pack
 			var found bool
 			if pack, found = packs[blob.pack]; !found {
-				pack = Pack{name: blob.pack, blobs: make(map[restic.ID]restic.Blob)}
+				pack = Pack{name: blob.pack, blobs: make(map[restic.ID]packBlobLayout)}
 			}
 
 			// calculate blob id and add to the pack as necessary
 			blobID := restic.Hash([]byte(blob.data))
 			if _, found := pack.blobs[blobID]; !found {
 				blobData := []byte(blob.data)
-				pack.blobs[blobID] = restic.Blob{
-					BlobHandle: restic.BlobHandle{
-						Type: restic.DataBlob,
-						ID:   blobID,
-					},
-					Length:             uint(len(blobData)),
-					UncompressedLength: uint(len(blobData)),
-					Offset:             uint(len(pack.data)),
+				n := uint(len(blobData))
+				pack.blobs[blobID] = packBlobLayout{
+					offset:     uint(len(pack.data)),
+					ciphertext: n,
+					plaintext:  n,
+					compressed: true,
 				}
 				pack.data = append(pack.data, blobData...)
 			}
@@ -110,14 +135,19 @@ func newTestRepo(content []TestFile) *TestRepo {
 		filesPathToContent[file.name] = content
 	}
 
-	blobs := make(map[restic.ID][]restic.PackedBlob)
+	blobs := make(map[restic.ID][]restic.PackBlob)
 	packsIDToData := make(map[restic.ID][]byte)
 
 	for _, pack := range packs {
 		packID := restic.Hash(pack.data)
 		packsIDToData[packID] = pack.data
-		for blobID, blob := range pack.blobs {
-			blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID})
+		for blobID, layout := range pack.blobs {
+			blobs[blobID] = append(blobs[blobID], &testPackBlob{
+				packID: packID,
+				handle: restic.BlobHandle{Type: restic.DataBlob, ID: blobID},
+				offset: layout.offset, ciphertext: layout.ciphertext,
+				plaintext: layout.plaintext, compressed: layout.compressed,
+			})
 		}
 	}
 
@@ -138,12 +168,12 @@ func newTestRepo(content []TestFile) *TestRepo {
 		warmupJobs:         []*TestWarmupJob{},
 	}
 	repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
-		entries := make([]restic.PackedBlob, 0, len(handles))
+		entries := make([]*testPackBlob, 0, len(handles))
 		for _, h := range handles {
 			found := false
 			for _, e := range repo.blobs[h.ID] {
-				if packID == e.PackID {
-					entries = append(entries, e)
+				if packID == e.PackID() {
+					entries = append(entries, e.(*testPackBlob))
 					found = true
 					break
 				}
@@ -152,13 +182,13 @@ func newTestRepo(content []TestFile) *TestRepo {
 				return fmt.Errorf("missing blob: %v", h)
 			}
 		}
-		slices.SortFunc(entries, func(a, b restic.PackedBlob) int {
-			return cmp.Compare(a.Offset, b.Offset)
+		slices.SortFunc(entries, func(a, b *testPackBlob) int {
+			return cmp.Compare(a.offset, b.offset)
 		})
 
 		for _, e := range entries {
-			buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length]
-			err := handleBlobFn(e.BlobHandle, buf, nil)
+			buf := repo.packsIDToData[packID][e.offset : e.offset+e.ciphertext]
+			err := handleBlobFn(e.handle, buf, nil)
 			if err != nil {
 				return err
 			}