amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-06-17 22:24:17 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #21848 from restic/shrink-repository-interface

Browse Source 
move restic.{Blob,Blobs,PackedBlob} types to `./internal/repository/pack` package
This commit is contained in:
Michael Eischer
2026-06-13 19:11:33 +02:00
committed by GitHub
parent 0938f52f38 c062a78dcd
commit f2ff05f840
41 changed files with 688 additions and 514 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/restic/cmd_copy.go
+5 -5
View File
@@ -189,7 +189,7 @@ func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool {
// copyTreeBatched copies multiple snapshots in one go. Snapshots are written after
// data equivalent to at least 10 packfiles was written.
func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
func copyTreeBatched(ctx context.Context, srcRepo *repository.Repository, dstRepo restic.Repository,
selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error {
// remember already processed trees across all snapshots
@@ -254,7 +254,7 @@ func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo res
return nil
}
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
func copyTree(ctx context.Context, srcRepo *repository.Repository, dstRepo restic.Repository,
visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) {
copyBlobs := srcRepo.NewAssociatedBlobSet()
@@ -268,7 +268,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
pb := srcRepo.LookupBlob(h.Type, h.ID)
copyBlobs.Insert(h)
for _, p := range pb {
packList.Insert(p.PackID)
packList.Insert(p.PackID())
}
}
}
@@ -317,9 +317,9 @@ func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, pa
countBlobs := 0
sizeBlobs := uint64(0)
for blob := range copyBlobs.Keys() {
for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) {
for _, pb := range srcRepo.LookupBlob(blob.Type, blob.ID) {
countBlobs++
sizeBlobs += uint64(blob.Length)
sizeBlobs += uint64(pb.CiphertextLength())
break
}
}
cmd/restic/cmd_copy_integration_test.go
+4 -4
View File
@@ -103,17 +103,17 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
defer unlock()
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
blobs, err := repo.ListPack(context.TODO(), id, size)
handles, err := repo.ListPackHandles(context.TODO(), id, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
rtest.Assert(t, len(handles) > 0, "a packfile should contain at least one blob")
switch blobs[0].Type {
switch handles[0].Type {
case restic.TreeBlob:
countTreePacks++
case restic.DataBlob:
countDataPacks++
}
countBlobs += len(blobs)
countBlobs += len(handles)
return nil
}))
return nil
cmd/restic/cmd_find.go
+15 -14
View File
@@ -473,18 +473,18 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
delete(packIDs, idStr)
}
debug.Log("Found pack %s", idStr)
blobs, err := f.repo.ListPack(ctx, id, size)
handles, err := f.repo.ListPackHandles(ctx, id, size)
if err != nil {
return err
}
for _, b := range blobs {
switch b.Type {
for _, h := range handles {
switch h.Type {
case restic.DataBlob:
f.blobIDs[b.ID.String()] = struct{}{}
f.blobIDs[h.ID.String()] = struct{}{}
case restic.TreeBlob:
f.treeIDs[b.ID.String()] = struct{}{}
f.treeIDs[h.ID.String()] = struct{}{}
default:
panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String()))
panic(fmt.Sprintf("unknown type %v in blob list", h.Type.String()))
}
}
// Stop searching when all packs have been found
@@ -526,22 +526,23 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
// remember which packs were found in the index
indexPackIDs := make(map[string]struct{})
err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) {
idStr := pb.PackID.String()
err := f.repo.ListBlobs(wctx, func(pb restic.PackBlob) {
packID := pb.PackID()
idStr := packID.String()
// keep entry in packIDs as Each() returns individual index entries
matchingID := false
if _, ok := packIDs[idStr]; ok {
matchingID = true
} else {
if _, ok := packIDs[pb.PackID.Str()]; ok {
if _, ok := packIDs[packID.Str()]; ok {
// expand id
delete(packIDs, pb.PackID.Str())
delete(packIDs, packID.Str())
packIDs[idStr] = struct{}{}
matchingID = true
}
}
if matchingID {
f.blobIDs[pb.ID.String()] = struct{}{}
f.blobIDs[pb.Handle().ID.String()] = struct{}{}
indexPackIDs[idStr] = struct{}{}
}
})
@@ -577,9 +578,9 @@ func (f *Finder) findObjectPack(id string, t restic.BlobType) {
}
for _, b := range blobs {
if b.ID.Equal(rid) {
f.printer.S("Object belongs to pack %s", b.PackID)
f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String())
if b.Handle().ID.Equal(rid) {
f.printer.S("Object belongs to pack %s", b.PackID())
f.printer.S(" ... Pack %s: %v", b.PackID().String(), b.Handle())
break
}
}
cmd/restic/cmd_find_integration_test.go
+8 -7
View File
@@ -179,9 +179,10 @@ func TestFindPackfile(t *testing.T) {
packID := restic.ID{}
done := false
err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) {
if !done && pb.Type == restic.TreeBlob {
packID = pb.PackID
err = repo.ListBlobs(ctx, func(pb restic.PackBlob) {
h := pb.Handle()
if !done && h.Type == restic.TreeBlob {
packID = pb.PackID()
done = true
}
})
@@ -236,12 +237,12 @@ func TestFindPackID(t *testing.T) {
// load Index
rtest.OK(t, repo.LoadIndex(ctx, nil))
// go through all index entries and collect data and tree packfile(s)
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
switch blob.Type {
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) {
switch blob.Handle().Type {
case restic.DataBlob:
dataPackID = blob.PackID
dataPackID = blob.PackID()
case restic.TreeBlob:
treePackID = blob.PackID
treePackID = blob.PackID()
}
}))
return nil
cmd/restic/cmd_list_integration_test.go
+2 -2
View File
@@ -71,8 +71,8 @@ func testListBlobs(t testing.TB, gopts global.Options) (blobSetFromIndex restic.
// get blobs from index
blobSetFromIndex = restic.NewIDSet()
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
blobSetFromIndex.Insert(blob.ID)
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) {
blobSetFromIndex.Insert(blob.Handle().ID)
}))
return nil
})
cmd/restic/cmd_recover.go
+4 -3
View File
@@ -75,9 +75,10 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
// tree. If it is not referenced, we have a root tree.
trees := make(map[restic.ID]bool)
err = repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
if blob.Type == restic.TreeBlob {
trees[blob.Blob.ID] = false
err = repo.ListBlobs(ctx, func(blob restic.PackBlob) {
h := blob.Handle()
if h.Type == restic.TreeBlob {
trees[h.ID] = false
}
})
if err != nil {
cmd/restic/cmd_stats.go
+7 -7
View File
@@ -160,16 +160,16 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
if len(pbs) == 0 {
return fmt.Errorf("blob %v not found", blobHandle)
}
stats.TotalSize += uint64(pbs[0].Length)
stats.TotalSize += uint64(pbs[0].CiphertextLength())
if repo.Config().Version >= 2 {
stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength())))
if pbs[0].IsCompressed() {
stats.TotalCompressedBlobsSize += uint64(pbs[0].Length)
stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
stats.TotalCompressedBlobsSize += uint64(pbs[0].CiphertextLength())
stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].PlaintextLength())))
}
}
stats.TotalBlobCount++
statsProgress.Update(0, 1, uint64(pbs[0].Length))
statsProgress.Update(0, 1, uint64(pbs[0].CiphertextLength()))
}
if stats.TotalCompressedBlobsSize > 0 {
stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize)
@@ -414,8 +414,8 @@ func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBl
hist[i] = newSizeHistogram(2 * chunker.MaxSize)
}
err := repo.ListBlobs(ctx, func(pb restic.PackedBlob) {
hist[pb.Type].Add(uint64(pb.Length))
err := repo.ListBlobs(ctx, func(pb restic.PackBlob) {
hist[pb.Handle().Type].Add(uint64(pb.CiphertextLength()))
})
return hist, err
cmd/restic/integration_helpers_test.go
+6 -6
View File
@@ -269,9 +269,9 @@ func listTreePacks(gopts global.Options, t *testing.T) restic.IDSet {
rtest.OK(t, r.LoadIndex(ctx, nil))
treePacks = restic.NewIDSet()
return r.ListBlobs(ctx, func(pb restic.PackedBlob) {
if pb.Type == restic.TreeBlob {
treePacks.Insert(pb.PackID)
return r.ListBlobs(ctx, func(pb restic.PackBlob) {
if pb.Handle().Type == restic.TreeBlob {
treePacks.Insert(pb.PackID())
}
})
})
@@ -318,9 +318,9 @@ func removePacksExcept(gopts global.Options, t testing.TB, keep restic.IDSet, re
rtest.OK(t, r.LoadIndex(ctx, nil))
treePacks := restic.NewIDSet()
rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) {
if pb.Type == restic.TreeBlob {
treePacks.Insert(pb.PackID)
rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackBlob) {
if pb.Handle().Type == restic.TreeBlob {
treePacks.Insert(pb.PackID())
}
}))
internal/checker/checker.go
+3 -3
View File
@@ -277,8 +277,8 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
err = c.repo.ListBlobs(ctx, func(blob restic.PackBlob) {
h := blob.Handle()
if !c.blobRefs.M.Has(h) {
debug.Log("blob %v not referenced", h)
blobs = append(blobs, h)
@@ -308,7 +308,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
// convert used blobs into their encompassing packfiles
for bh := range c.blobRefs.M.Keys() {
for _, pb := range c.repo.LookupBlob(bh.Type, bh.ID) {
filteredPacks[pb.PackID] = allPacks[pb.PackID]
filteredPacks[pb.PackID()] = allPacks[pb.PackID()]
}
}
internal/repository/checker.go
+17 -14
View File
@@ -87,16 +87,18 @@ func NewChecker(repo *Repository) *Checker {
}
func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) {
packs := make(map[restic.ID]restic.BlobType)
err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) {
tpe, exists := packs[pb.PackID]
err := idx.ListBlobs(ctx, func(pb restic.PackBlob) {
packID := pb.PackID()
h := pb.Handle()
tpe, exists := packs[packID]
if exists {
if pb.Type != tpe {
if h.Type != tpe {
tpe = restic.InvalidBlob
}
} else {
tpe = pb.Type
tpe = h.Type
}
packs[pb.PackID] = tpe
packs[packID] = tpe
})
return packs, err
}
@@ -127,10 +129,11 @@ func (c *Checker) LoadIndex(ctx context.Context, p restic.TerminalCounterFactory
}
cnt++
if _, ok := packToIndex[blob.PackID]; !ok {
packToIndex[blob.PackID] = restic.NewIDSet()
packID := blob.PackID()
if _, ok := packToIndex[packID]; !ok {
packToIndex[packID] = restic.NewIDSet()
}
packToIndex[blob.PackID].Insert(id)
packToIndex[packID].Insert(id)
}
for pbs := range idx.EachByPack(ctx, restic.NewIDSet()) {
@@ -257,7 +260,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
type checkTask struct {
id restic.ID
size int64
blobs restic.Blobs
blobs pack.Blobs
}
ch := make(chan checkTask)
@@ -306,7 +309,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
}
// push packs to ch
for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) {
for pbs := range c.repo.listPacksFromIndex(ctx, packSet) {
size := packs[pbs.PackID]
debug.Log("listed %v", pbs.PackID)
select {
@@ -327,7 +330,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
}
// checkPack reads a pack and checks the integrity of all blobs.
func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err != nil {
if r.cache != nil {
@@ -346,7 +349,7 @@ func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Bl
return err
}
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
type partialReadError struct {
error
@@ -462,8 +465,8 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest
for _, blob := range blobs {
// Check if blob is contained in index and position is correct
idxHas := false
for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
if pb.PackID == id && pb.Blob == blob {
for _, pb := range r.idx.Lookup(blob.BlobHandle) {
if pb.PackID().Equal(id) && pb.Blob == blob {
idxHas = true
break
}
internal/repository/checker_test.go
+3 -3
View File
@@ -21,7 +21,7 @@ import (
var checkerTestData = filepath.Join("..", "checker", "testdata", "checker-test-repo.tar.gz")
func testWrapCheckPack(ctx context.Context, t *testing.T, repo *Repository,
packID restic.ID, blobs []restic.Blob, size int64,
packID restic.ID, blobs pack.Blobs, size int64,
) error {
t.Helper()
bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize)
@@ -46,8 +46,8 @@ func TestGapInBlobs(t *testing.T) {
_, ok := repoPacks[packID]
rtest.Assert(t, ok, "expected pack 19a731a515618ec8b75fc0ff3b887d8feb83aef1001c9899f6702761142ed068")
blobs := []restic.Blob{}
pb := <-repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(packID))
blobs := pack.Blobs{}
pb := <-repo.listPacksFromIndex(context.TODO(), restic.NewIDSet(packID))
blobs = append(blobs, pb.Blobs...)
// assertion for clarity, actually can't fail as the packfile content is fixed
internal/repository/debug.go
+8 -9
View File
@@ -49,7 +49,7 @@ func writePackDumpJSON(wr io.Writer, item any) error {
func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, err := repo.ListPack(ctx, id, size)
blobs, err := repo.listPack(ctx, id, size)
if err != nil {
printer.E("error for pack %v: %v", id.Str(), err)
return nil
@@ -116,15 +116,14 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami
blobsLoaded := false
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
for b := range repo.listPacksFromIndex(ctx, restic.NewIDSet(id)) {
if len(b.Blobs) == 0 {
continue
}
checkPackSize(blobs, len(buf), printer)
checkPackSize(b.Blobs, len(buf), printer)
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
err := loadBlobs(ctx, opts, repo, id, b.Blobs, printer)
if err != nil {
printer.E("error: %v", err)
} else {
@@ -135,7 +134,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami
printer.S(" ========================================")
printer.S(" inspect the pack itself")
blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
blobs, err := repo.listPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
@@ -147,7 +146,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami
return nil
}
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
func checkPackSize(blobs pack.Blobs, fileSize int, printer progress.Printer) {
// track current size and offset
var size, offset uint64
@@ -285,7 +284,7 @@ func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
return out
}
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list pack.Blobs, printer progress.Printer) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
internal/repository/index/associated_data.go
+4 -3
View File
@@ -159,14 +159,15 @@ func (a *AssociatedSet[T]) All() iter.Seq2[restic.BlobHandle, T] {
}
for pb := range a.idx.Values() {
if _, ok := a.overflow[pb.BlobHandle]; ok {
bh := pb.Handle()
if _, ok := a.overflow[bh]; ok {
// already reported via overflow set
continue
}
val, known := a.Get(pb.BlobHandle)
val, known := a.Get(bh)
if known {
if !yield(pb.BlobHandle, val) {
if !yield(bh, val) {
return
}
}
internal/repository/index/associated_data_test.go
+13 -12
View File
@@ -6,6 +6,7 @@ import (
"testing"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@@ -19,17 +20,17 @@ func (n *noopSaver) SaveUnpacked(_ context.Context, _ restic.FileType, buf []byt
return restic.Hash(buf), nil
}
func makeFakePackedBlob() (restic.BlobHandle, restic.PackedBlob) {
func makeFakePackedBlob() (restic.BlobHandle, *pack.PackedBlob) {
bh := restic.NewRandomBlobHandle()
blob := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
pb := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bh,
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
return bh, blob
return bh, pb
}
func list(bs *AssociatedSet[uint8]) restic.BlobHandles {
@@ -40,7 +41,7 @@ func TestAssociatedSet(t *testing.T) {
bh, blob := makeFakePackedBlob()
mi := NewMasterIndex()
test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{}))
test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{}))
test.OK(t, mi.Flush(context.TODO(), &noopSaver{}))
bs := NewAssociatedSet[uint8](mi)
@@ -123,14 +124,14 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) {
_, blob := makeFakePackedBlob()
mi := NewMasterIndex()
test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{}))
test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{}))
test.OK(t, mi.Flush(context.TODO(), &noopSaver{}))
bs := NewAssociatedSet[uint8](mi)
// add new blobs to index after building the set
of, blob2 := makeFakePackedBlob()
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, &noopSaver{}))
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, &noopSaver{}))
test.OK(t, mi.Flush(context.TODO(), &noopSaver{}))
// non-existent
@@ -167,10 +168,10 @@ func TestAssociatedSetIntersectAndSub(t *testing.T) {
bh3, blob3 := makeFakePackedBlob()
bh4, blob4 := makeFakePackedBlob()
test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, restic.Blobs{blob3.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, restic.Blobs{blob4.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob3.PackID(), pack.Blobs{blob3.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob4.PackID(), pack.Blobs{blob4.Blob}, saver))
test.OK(t, mi.Flush(context.TODO(), saver))
t.Run("Intersect", func(t *testing.T) {
internal/repository/index/index.go
+25 -18
View File
@@ -73,7 +73,7 @@ func (idx *Index) addToPacks(id restic.ID) int {
return len(idx.packs) - 1
}
func (idx *Index) store(packIndex int, blob restic.Blob) {
func (idx *Index) store(packIndex int, blob pack.Blob) {
// assert that offset and length fit into uint32!
if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 {
panic("offset or length does not fit in uint32. You have packs > 4GB!")
@@ -146,7 +146,7 @@ func (idx *Index) Preallocate(t restic.BlobType, numEntries int) {
// StorePack remembers the ids of all blobs of a given pack
// in the index
func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) {
func (idx *Index) StorePack(id restic.ID, blobs pack.Blobs) {
idx.m.Lock()
defer idx.m.Unlock()
@@ -162,23 +162,24 @@ func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) {
}
}
func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) restic.PackedBlob {
return restic.PackedBlob{
Blob: restic.Blob{
func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) *pack.PackedBlob {
return &pack.PackedBlob{
Pack: idx.packs[e.packIndex],
Blob: pack.Blob{
BlobHandle: restic.BlobHandle{
ID: e.id,
Type: t},
Type: t,
},
Length: uint(e.length),
Offset: uint(e.offset),
UncompressedLength: uint(e.uncompressedLength),
},
PackID: idx.packs[e.packIndex],
}
}
// Lookup queries the index for the blob ID and returns all entries including
// duplicates. Adds found entries to blobs and returns the result.
func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic.PackedBlob {
// duplicates. Adds found entries to pbs and returns the result.
func (idx *Index) Lookup(bh restic.BlobHandle, pbs []*pack.PackedBlob) []*pack.PackedBlob {
idx.m.RLock()
defer idx.m.RUnlock()
@@ -215,8 +216,8 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found
// Values returns an iterator over all blobs known to the index. This blocks any
// modification of the index.
func (idx *Index) Values() iter.Seq[restic.PackedBlob] {
return func(yield func(restic.PackedBlob) bool) {
func (idx *Index) Values() iter.Seq[*pack.PackedBlob] {
return func(yield func(*pack.PackedBlob) bool) {
idx.m.RLock()
defer idx.m.RUnlock()
@@ -231,17 +232,23 @@ func (idx *Index) Values() iter.Seq[restic.PackedBlob] {
}
}
// EachByPack returns a channel that yields all blobs known to the index
// PackBlobs lists all blobs contained in a pack file according to the index.
type PackBlobs struct {
PackID restic.ID
Blobs pack.Blobs
}
// EachByPack returns a channel that yields all blobs known to the index,
// grouped by packID but ignoring blobs with a packID in packPlacklist for
// finalized indexes.
// This filtering is used when rebuilding the index where we need to ignore packs
// from the finalized index which have been re-read into a non-finalized index.
// When the context is cancelled, the background goroutine
// terminates. This blocks any modification of the index.
func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan restic.PackBlobs {
func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan PackBlobs {
idx.m.RLock()
ch := make(chan restic.PackBlobs)
ch := make(chan PackBlobs)
go func() {
defer idx.m.RUnlock()
@@ -262,7 +269,7 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-
}
for packID, packByType := range byPack {
var result restic.PackBlobs
var result PackBlobs
result.PackID = packID
for typ, p := range packByType {
for _, e := range p {
@@ -482,7 +489,7 @@ func (idx *Index) merge(idx2 *Index) error {
for e := range m.valuesWithID(e2.id) {
b := idx.toPackedBlob(e, restic.BlobType(typ))
b2 := idx2.toPackedBlob(e2, restic.BlobType(typ))
if b == b2 {
if *b == *b2 {
found = true
break
}
@@ -519,7 +526,7 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, err error) {
packID := idx.addToPacks(p.ID)
for _, blob := range p.Blobs {
idx.store(packID, restic.Blob{
idx.store(packID, pack.Blob{
BlobHandle: restic.BlobHandle{
Type: blob.Type,
ID: blob.ID},
@@ -550,7 +557,7 @@ func (idx *Index) Len(t restic.BlobType) uint {
return idx.byType[t].len()
}
func PackBlobsHash(pbs restic.PackBlobs) restic.ID {
func PackBlobsHash(pbs PackBlobs) restic.ID {
h := sha256.New()
h.Write(pbs.PackID[:])
internal/repository/index/index_internal_test.go
+2 -2
View File
@@ -14,7 +14,7 @@ func TestIndexOversized(t *testing.T) {
// Add blobs up to indexMaxBlobs + pack.MaxHeaderEntries - 1
packID := idx.addToPacks(restic.NewRandomID())
for i := uint(0); i < indexMaxBlobs+pack.MaxHeaderEntries-1; i++ {
idx.store(packID, restic.Blob{
idx.store(packID, pack.Blob{
BlobHandle: restic.BlobHandle{
Type: restic.DataBlob,
ID: restic.NewRandomID(),
@@ -27,7 +27,7 @@ func TestIndexOversized(t *testing.T) {
rtest.Assert(t, !Oversized(idx), "index should not be considered oversized")
// Add one more blob to exceed the limit
idx.store(packID, restic.Blob{
idx.store(packID, pack.Blob{
BlobHandle: restic.BlobHandle{
Type: restic.DataBlob,
ID: restic.NewRandomID(),
internal/repository/index/index_test.go
+44 -43
View File
@@ -9,19 +9,20 @@ import (
"testing"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestIndexSerialize(t *testing.T) {
tests := []restic.PackedBlob{}
tests := []*pack.PackedBlob{}
idx := index.NewIndex()
// create 50 packs with 20 blobs each
for i := 0; i < 50; i++ {
packID := restic.NewRandomID()
var blobs restic.Blobs
var blobs pack.Blobs
pos := uint(0)
for j := 0; j < 20; j++ {
@@ -31,14 +32,14 @@ func TestIndexSerialize(t *testing.T) {
// test a mix of compressed and uncompressed packs
uncompressedLength = 2 * length
}
pb := restic.PackedBlob{
Blob: restic.Blob{
pb := &pack.PackedBlob{
Pack: packID,
Blob: pack.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Offset: pos,
Length: length,
UncompressedLength: uncompressedLength,
},
PackID: packID,
}
blobs = append(blobs, pb.Blob)
tests = append(tests, pb)
@@ -64,17 +65,17 @@ func TestIndexSerialize(t *testing.T) {
rtest.OK(t, err)
for _, testBlob := range tests {
list := idx.Lookup(testBlob.BlobHandle, nil)
list := idx.Lookup(testBlob.Handle(), nil)
if len(list) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list)
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list)
}
result := list[0]
rtest.Equals(t, testBlob, result)
list2 := idx2.Lookup(testBlob.BlobHandle, nil)
list2 := idx2.Lookup(testBlob.Handle(), nil)
if len(list2) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list2), list2)
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list2), list2)
}
result2 := list2[0]
@@ -82,21 +83,21 @@ func TestIndexSerialize(t *testing.T) {
}
// add more blobs to idx
newtests := []restic.PackedBlob{}
newtests := []*pack.PackedBlob{}
for i := 0; i < 10; i++ {
packID := restic.NewRandomID()
var blobs restic.Blobs
var blobs pack.Blobs
pos := uint(0)
for j := 0; j < 10; j++ {
length := uint(i*100 + j)
pb := restic.PackedBlob{
Blob: restic.Blob{
pb := &pack.PackedBlob{
Pack: packID,
Blob: pack.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Offset: pos,
Length: length,
},
PackID: packID,
}
blobs = append(blobs, pb.Blob)
newtests = append(newtests, pb)
@@ -127,9 +128,9 @@ func TestIndexSerialize(t *testing.T) {
// all new blobs must be in the index
for _, testBlob := range newtests {
list := idx3.Lookup(testBlob.BlobHandle, nil)
list := idx3.Lookup(testBlob.Handle(), nil)
if len(list) != 1 {
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list)
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list)
}
blob := list[0]
@@ -145,12 +146,12 @@ func TestIndexSize(t *testing.T) {
blobCount := 100
for i := 0; i < packs; i++ {
packID := restic.NewRandomID()
var blobs restic.Blobs
var blobs pack.Blobs
pos := uint(0)
for j := 0; j < blobCount; j++ {
length := uint(i*100 + j)
blobs = append(blobs, restic.Blob{
blobs = append(blobs, pack.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Offset: pos,
Length: length,
@@ -293,15 +294,15 @@ func TestIndexUnserialize(t *testing.T) {
t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob)
rtest.Equals(t, test.packID, blob.PackID)
rtest.Equals(t, test.tpe, blob.Type)
rtest.Equals(t, test.offset, blob.Offset)
rtest.Equals(t, test.length, blob.Length)
rtest.Equals(t, test.packID, blob.PackID())
rtest.Equals(t, test.tpe, blob.Blob.Type)
rtest.Equals(t, test.offset, blob.Blob.Offset)
rtest.Equals(t, test.length, blob.Blob.Length)
switch task.version {
case 1:
rtest.Equals(t, uint(0), blob.UncompressedLength)
rtest.Equals(t, uint(0), blob.Blob.UncompressedLength)
case 2:
rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength)
rtest.Equals(t, test.uncompressedLength, blob.Blob.UncompressedLength)
default:
t.Fatal("Invalid index version")
}
@@ -313,20 +314,20 @@ func TestIndexUnserialize(t *testing.T) {
}
for _, blob := range blobs {
b, ok := exampleLookupTest.blobs[blob.ID]
b, ok := exampleLookupTest.blobs[blob.Handle().ID]
if !ok {
t.Errorf("unexpected blob %v found", blob.ID.Str())
t.Errorf("unexpected blob %v found", blob.Handle().ID.String())
}
if blob.Type != b {
t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type)
if blob.Blob.Type != b {
t.Errorf("unexpected type for blob %v: want %v, got %v", blob.Handle().ID.String(), b, blob.Blob.Type)
}
}
}
}
func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []*pack.PackedBlob) {
for pb := range idx.Values() {
if pb.PackID.Equal(id) {
if pb.PackID().Equal(id) {
pbs = append(pbs, pb)
}
}
@@ -401,7 +402,7 @@ func TestIndexPacks(t *testing.T) {
for i := 0; i < 20; i++ {
packID := restic.NewRandomID()
idx.StorePack(packID, restic.Blobs{
idx.StorePack(packID, pack.Blobs{
{
BlobHandle: restic.NewRandomBlobHandle(),
Offset: 0,
@@ -433,12 +434,12 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupB
// create index with given number of pack files
for i := 0; i < packfiles; i++ {
packID := NewRandomTestID(rng)
var blobs restic.Blobs
var blobs pack.Blobs
offset := 0
for offset < maxPackSize {
size := 2000 + rng.Intn(4*1024*1024)
id := NewRandomTestID(rng)
blobs = append(blobs, restic.Blob{
blobs = append(blobs, pack.Blob{
BlobHandle: restic.BlobHandle{
Type: restic.DataBlob,
ID: id,
@@ -482,7 +483,7 @@ func BenchmarkIndexHasKnown(b *testing.B) {
idx, _ := createRandomIndex(rand.New(rand.NewSource(0)), 200000)
handles := make([]restic.BlobHandle, 0, 100000)
for handle := range idx.Values() {
handles = append(handles, handle.BlobHandle)
handles = append(handles, handle.Handle())
if len(handles) == cap(handles) {
break
}
@@ -517,14 +518,14 @@ func BenchmarkIndexAllocParallel(b *testing.B) {
}
func TestIndexHas(t *testing.T) {
tests := []restic.PackedBlob{}
tests := []*pack.PackedBlob{}
idx := index.NewIndex()
// create 50 packs with 20 blobs each
for i := 0; i < 50; i++ {
packID := restic.NewRandomID()
var blobs restic.Blobs
var blobs pack.Blobs
pos := uint(0)
for j := 0; j < 20; j++ {
@@ -534,14 +535,14 @@ func TestIndexHas(t *testing.T) {
// test a mix of compressed and uncompressed packs
uncompressedLength = 2 * length
}
pb := restic.PackedBlob{
Blob: restic.Blob{
pb := &pack.PackedBlob{
Pack: packID,
Blob: pack.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Offset: pos,
Length: length,
UncompressedLength: uncompressedLength,
},
PackID: packID,
}
blobs = append(blobs, pb.Blob)
tests = append(tests, pb)
@@ -551,11 +552,11 @@ func TestIndexHas(t *testing.T) {
}
for _, testBlob := range tests {
rtest.Assert(t, idx.Has(testBlob.BlobHandle), "Index reports not having data blob added to it")
rtest.Assert(t, idx.Has(testBlob.Handle()), "Index reports not having data blob added to it")
}
rtest.Assert(t, !idx.Has(restic.NewRandomBlobHandle()), "Index reports having a data blob not added to it")
rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob")
rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].Handle().ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob")
}
func TestMixedEachByPack(t *testing.T) {
@@ -566,7 +567,7 @@ func TestMixedEachByPack(t *testing.T) {
for i := 0; i < 50; i++ {
packID := restic.NewRandomID()
expected[packID] = 1
blobs := restic.Blobs{
blobs := pack.Blobs{
{
BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()},
Offset: 0,
@@ -608,7 +609,7 @@ func TestEachByPackIgnoes(t *testing.T) {
} else {
expected[packID] = 1
}
blobs := restic.Blobs{
blobs := pack.Blobs{
{
BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()},
Offset: 0,
internal/repository/index/master_index.go
+14 -11
View File
@@ -8,6 +8,7 @@ import (
"sync"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup"
@@ -39,10 +40,11 @@ func (mi *MasterIndex) clearPendingBlobs() {
}
// Lookup queries all known Indexes for the ID and returns all matches.
func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) {
func (mi *MasterIndex) Lookup(bh restic.BlobHandle) []*pack.PackedBlob {
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
var pbs []*pack.PackedBlob
for _, idx := range mi.idx {
pbs = idx.Lookup(bh, pbs)
}
@@ -145,12 +147,12 @@ func (mi *MasterIndex) Insert(idx *Index) {
}
// StorePack remembers the id and pack in the index.
func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs restic.Blobs, r restic.SaverUnpacked[restic.FileType]) error {
func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs pack.Blobs, r restic.SaverUnpacked[restic.FileType]) error {
mi.storePack(id, blobs)
return mi.saveFullIndex(ctx, r)
}
func (mi *MasterIndex) storePack(id restic.ID, blobs restic.Blobs) {
func (mi *MasterIndex) storePack(id restic.ID, blobs pack.Blobs) {
mi.idxMutex.Lock()
defer mi.idxMutex.Unlock()
@@ -218,8 +220,8 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index {
// Values returns an iterator over all blobs known to the index. This blocks any
// modification of the index.
func (mi *MasterIndex) Values() iter.Seq[restic.PackedBlob] {
return func(yield func(restic.PackedBlob) bool) {
func (mi *MasterIndex) Values() iter.Seq[*pack.PackedBlob] {
return func(yield func(*pack.PackedBlob) bool) {
mi.idxMutex.RLock()
defer mi.idxMutex.RUnlock()
@@ -663,13 +665,13 @@ func (mi *MasterIndex) saveFullIndex(ctx context.Context, r restic.SaverUnpacked
}
// ListPacks returns the blobs of the specified pack files grouped by pack file.
func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs {
out := make(chan restic.PackBlobs)
func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan PackBlobs {
out := make(chan PackBlobs)
go func() {
defer close(out)
// only resort a part of the index to keep the memory overhead bounded
for i := byte(0); i < 16; i++ {
packBlob := make(map[restic.ID]restic.Blobs)
packBlob := make(map[restic.ID]pack.Blobs)
for pack := range packs {
if pack[0]&0xf == i {
packBlob[pack] = nil
@@ -682,8 +684,9 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
if ctx.Err() != nil {
return
}
if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i {
packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob)
packID := pb.PackID()
if packs.Has(packID) && packID[0]&0xf == i {
packBlob[packID] = append(packBlob[packID], pb.Blob)
}
}
@@ -692,7 +695,7 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
// allow GC
packBlob[packID] = nil
select {
case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}:
case out <- PackBlobs{PackID: packID, Blobs: pbs}:
case <-ctx.Done():
return
}
internal/repository/index/master_index_test.go
+93 -91
View File
@@ -14,6 +14,7 @@ import (
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/progress"
@@ -24,18 +25,18 @@ func TestMasterIndex(t *testing.T) {
bhInIdx2 := restic.NewRandomBlobHandle()
bhInIdx12 := restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob}
blob1 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blob1 := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bhInIdx1,
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
blob2 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blob2 := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bhInIdx2,
Length: uint(crypto.CiphertextLength(100)),
Offset: 10,
@@ -43,9 +44,9 @@ func TestMasterIndex(t *testing.T) {
},
}
blob12a := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blob12a := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bhInIdx12,
Length: uint(crypto.CiphertextLength(123)),
Offset: 110,
@@ -53,9 +54,9 @@ func TestMasterIndex(t *testing.T) {
},
}
blob12b := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blob12b := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bhInIdx12,
Length: uint(crypto.CiphertextLength(123)),
Offset: 50,
@@ -64,12 +65,12 @@ func TestMasterIndex(t *testing.T) {
}
idx1 := index.NewIndex()
idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob})
idx1.StorePack(blob12a.PackID, restic.Blobs{blob12a.Blob})
idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob})
idx1.StorePack(blob12a.PackID(), pack.Blobs{blob12a.Blob})
idx2 := index.NewIndex()
idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob})
idx2.StorePack(blob12b.PackID, restic.Blobs{blob12b.Blob})
idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob})
idx2.StorePack(blob12b.PackID(), pack.Blobs{blob12b.Blob})
mIdx := index.NewMasterIndex()
mIdx.Insert(idx1)
@@ -77,7 +78,7 @@ func TestMasterIndex(t *testing.T) {
// test idInIdx1
blobs := mIdx.Lookup(bhInIdx1)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs)
size, found := mIdx.LookupSize(bhInIdx1)
rtest.Equals(t, true, found)
@@ -85,7 +86,7 @@ func TestMasterIndex(t *testing.T) {
// test idInIdx2
blobs = mIdx.Lookup(bhInIdx2)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs)
size, found = mIdx.LookupSize(bhInIdx2)
rtest.Equals(t, true, found)
@@ -95,19 +96,20 @@ func TestMasterIndex(t *testing.T) {
blobs = mIdx.Lookup(bhInIdx12)
rtest.Equals(t, 2, len(blobs))
// test Lookup result for blob12a
found = false
if blobs[0] == blob12a || blobs[1] == blob12a {
found = true
containsPackedBlob := func(list []*pack.PackedBlob, want *pack.PackedBlob) bool {
for _, b := range list {
if b.PackID().Equal(want.PackID()) && b.Blob == want.Blob {
return true
}
}
return false
}
rtest.Assert(t, found, "blob12a not found in result")
// test Lookup result for blob12a
rtest.Assert(t, containsPackedBlob(blobs, blob12a), "blob12a not found in result")
// test Lookup result for blob12b
found = false
if blobs[0] == blob12b || blobs[1] == blob12b {
found = true
}
rtest.Assert(t, found, "blob12a not found in result")
rtest.Assert(t, containsPackedBlob(blobs, blob12b), "blob12b not found in result")
size, found = mIdx.LookupSize(bhInIdx12)
rtest.Equals(t, true, found)
@@ -135,7 +137,7 @@ func TestMasterIndexAddPending(t *testing.T) {
// Test AddPending: try to add a blob that's already in an index (should return false)
bhInIndex := restic.NewRandomBlobHandle()
idx := index.NewIndex()
idx.StorePack(restic.NewRandomID(), restic.Blobs{{
idx.StorePack(restic.NewRandomID(), pack.Blobs{{
BlobHandle: bhInIndex,
Length: uint(crypto.CiphertextLength(50)),
Offset: 0,
@@ -173,14 +175,14 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) {
// Store the blob in a pack
packID := restic.NewRandomID()
blob := restic.Blob{
blob := pack.Blob{
BlobHandle: bhPending,
Length: uint(crypto.CiphertextLength(75)),
Offset: 0,
UncompressedLength: 75,
}
saver := &noopSaver{}
err := mIdx.StorePack(context.Background(), packID, restic.Blobs{blob}, saver)
err := mIdx.StorePack(context.Background(), packID, pack.Blobs{blob}, saver)
rtest.OK(t, err)
// Verify it is still found
@@ -191,8 +193,8 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) {
// Verify the blob can be found via Lookup from the index
blobs := mIdx.Lookup(bhPending)
rtest.Assert(t, len(blobs) > 0, "blob should be found in index after StorePack")
rtest.Equals(t, packID, blobs[0].PackID)
rtest.Equals(t, bhPending, blobs[0].BlobHandle)
rtest.Equals(t, packID, blobs[0].PackID())
rtest.Equals(t, bhPending, blobs[0].Handle())
// Test that adding the same blob as pending again fails (it's now in index)
added = mIdx.AddPending(bhPending, 100)
@@ -203,18 +205,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
bhInIdx1 := restic.NewRandomBlobHandle()
bhInIdx2 := restic.NewRandomBlobHandle()
blob1 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blob1 := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bhInIdx1,
Length: 10,
Offset: 0,
},
}
blob2 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blob2 := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bhInIdx2,
Length: 100,
Offset: 10,
@@ -223,10 +225,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
}
idx1 := index.NewIndex()
idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob})
idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob})
idx2 := index.NewIndex()
idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob})
idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob})
mIdx := index.NewMasterIndex()
mIdx.Insert(idx1)
@@ -246,18 +248,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
rtest.Equals(t, 2, blobCount)
blobs := mIdx.Lookup(bhInIdx1)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs)
blobs = mIdx.Lookup(bhInIdx2)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs)
blobs = mIdx.Lookup(restic.NewRandomBlobHandle())
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
// merge another index containing identical blobs
idx3 := index.NewIndex()
idx3.StorePack(blob1.PackID, restic.Blobs{blob1.Blob})
idx3.StorePack(blob2.PackID, restic.Blobs{blob2.Blob})
idx3.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob})
idx3.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob})
mIdx.Insert(idx3)
finalIndexes, idxCount, newIDs := index.TestMergeIndex(t, mIdx)
@@ -268,10 +270,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
// Index should have same entries as before!
blobs = mIdx.Lookup(bhInIdx1)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs)
blobs = mIdx.Lookup(bhInIdx2)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs)
blobCount = 0
for range mIdx.Values() {
@@ -448,9 +450,9 @@ func testIndexSave(t *testing.T, version uint) {
idx := index.NewMasterIndex()
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
blobs := make(map[restic.PackedBlob]struct{})
blobs := make(map[pack.PackedBlob]struct{})
for pb := range idx.Values() {
blobs[pb] = struct{}{}
blobs[*pb] = struct{}{}
}
rtest.OK(t, test.saver(idx, unpacked))
@@ -458,8 +460,8 @@ func testIndexSave(t *testing.T, version uint) {
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
for pb := range idx.Values() {
if _, ok := blobs[pb]; ok {
delete(blobs, pb)
if _, ok := blobs[*pb]; ok {
delete(blobs, *pb)
} else {
t.Fatalf("unexpected blobs %v", pb)
}
@@ -481,9 +483,9 @@ func testIndexSavePartial(t *testing.T, version uint) {
// capture blob list before adding fourth snapshot
idx := index.NewMasterIndex()
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
blobs := make(map[restic.PackedBlob]struct{})
blobs := make(map[pack.PackedBlob]struct{})
for pb := range idx.Values() {
blobs[pb] = struct{}{}
blobs[*pb] = struct{}{}
}
// add+remove new snapshot and track its pack files
@@ -502,8 +504,8 @@ func testIndexSavePartial(t *testing.T, version uint) {
idx = index.NewMasterIndex()
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
for pb := range idx.Values() {
if _, ok := blobs[pb]; ok {
delete(blobs, pb)
if _, ok := blobs[*pb]; ok {
delete(blobs, *pb)
} else {
t.Fatalf("unexpected blobs %v", pb)
}
@@ -516,7 +518,7 @@ func testIndexSavePartial(t *testing.T, version uint) {
checker.TestCheckRepo(t, repo)
}
func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[restic.PackedBlob]struct{} {
func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[pack.PackedBlob]struct{} {
p := progress.NewCounter(0, 0, nil)
rtest.OK(t, master.Load(context.TODO(), repo, p, nil))
v, max := p.Get()
@@ -525,10 +527,10 @@ func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, ma
return collectBlobs(master)
}
func collectBlobs(master *index.MasterIndex) map[restic.PackedBlob]struct{} {
s := make(map[restic.PackedBlob]struct{})
func collectBlobs(master *index.MasterIndex) map[pack.PackedBlob]struct{} {
s := make(map[pack.PackedBlob]struct{})
for pb := range master.Values() {
s[pb] = struct{}{}
s[*pb] = struct{}{}
}
return s
}
@@ -588,17 +590,17 @@ func TestRewriteOversizedIndex(t *testing.T) {
return idx.Len(restic.DataBlob) > 2*fullIndexCount
}
var blobs restic.Blobs
var blobs pack.Blobs
// build oversized index
idx := index.NewIndex()
numPacks := 5
for p := 0; p < numPacks; p++ {
packID := restic.NewRandomID()
packBlobs := make(restic.Blobs, 0, fullIndexCount)
packBlobs := make(pack.Blobs, 0, fullIndexCount)
for i := 0; i < fullIndexCount; i++ {
blob := restic.Blob{
blob := pack.Blob{
BlobHandle: restic.BlobHandle{
Type: restic.DataBlob,
ID: restic.NewRandomID(),
@@ -645,17 +647,17 @@ func TestRewriteSplitPacks(t *testing.T) {
bh2 := restic.NewRandomBlobHandle()
bhOther := restic.NewRandomBlobHandle()
blob1 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blob1 := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bh1,
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
blob2 := restic.PackedBlob{
PackID: blob1.PackID,
Blob: restic.Blob{
blob2 := &pack.PackedBlob{
Pack: blob1.PackID(),
Blob: pack.Blob{
BlobHandle: bh2,
Length: uint(crypto.CiphertextLength(100)),
Offset: 10,
@@ -663,9 +665,9 @@ func TestRewriteSplitPacks(t *testing.T) {
},
}
// used to force index repacking
blobOther := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
blobOther := &pack.PackedBlob{
Pack: restic.NewRandomID(),
Blob: pack.Blob{
BlobHandle: bhOther,
Length: uint(crypto.CiphertextLength(100)),
Offset: 10,
@@ -673,25 +675,25 @@ func TestRewriteSplitPacks(t *testing.T) {
}
mi := index.NewMasterIndex()
rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID), nil, nil, index.MasterIndexRewriteOpts{}))
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID()), nil, nil, index.MasterIndexRewriteOpts{}))
mi = index.NewMasterIndex()
rtest.OK(t, mi.Load(context.TODO(), repo, nil, nil))
// test that all blobs are still in the index
for _, blob := range []restic.PackedBlob{blob1, blob2} {
blobs := mi.Lookup(blob.BlobHandle)
rtest.Equals(t, []restic.PackedBlob{blob}, blobs)
for _, blob := range []*pack.PackedBlob{blob1, blob2} {
blobs := mi.Lookup(blob.Handle())
rtest.Equals(t, []*pack.PackedBlob{blob}, blobs)
}
blobs := mi.Lookup(blobOther.BlobHandle)
blobs := mi.Lookup(blobOther.Handle())
rtest.Equals(t, nil, blobs)
}
@@ -713,17 +715,17 @@ func TestRewriteFullPacks(t *testing.T) {
packA := restic.NewRandomID()
packB := restic.NewRandomID()
blobA := restic.PackedBlob{
PackID: packA,
Blob: restic.Blob{
blobA := &pack.PackedBlob{
Pack: packA,
Blob: pack.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
blobB := restic.PackedBlob{
PackID: packB,
Blob: restic.Blob{
blobB := &pack.PackedBlob{
Pack: packB,
Blob: pack.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(50)),
Offset: 0,
@@ -731,11 +733,11 @@ func TestRewriteFullPacks(t *testing.T) {
}
mi := index.NewMasterIndex()
rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packA, pack.Blobs{blobA.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
indexIDs := mi.IDs()
@@ -750,6 +752,6 @@ func TestRewriteFullPacks(t *testing.T) {
rtest.Equals(t, 2, len(afterRewrite))
rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
rtest.Equals(t, []*pack.PackedBlob{blobA}, mi2.Lookup(blobA.Handle()))
rtest.Equals(t, []*pack.PackedBlob{blobB}, mi2.Lookup(blobB.Handle()))
}
internal/repository/index_list.go
+1 -1
View File
@@ -28,7 +28,7 @@ func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.Load
if ctx.Err() != nil {
return ctx.Err()
}
if !yield(IndexBlob{Handle: blob.BlobHandle}) {
if !yield(IndexBlob{Handle: blob.Handle()}) {
return stopIteration
}
}
internal/repository/index_list_test.go
+2 -2
View File
@@ -26,8 +26,8 @@ func TestAllIndexBlobs(t *testing.T) {
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
fromMaster := restic.NewBlobSet()
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
fromMaster.Insert(pb.BlobHandle)
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
fromMaster.Insert(pb.Handle())
}))
rtest.Equals(t, want, fromMaster)
internal/repository/index_testutil_test.go
+18
View File
@@ -0,0 +1,18 @@
package repository
import (
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
)
// BlobsInPack returns index entries for blobs stored in packID, sorted by offset.
func BlobsInPack(repo *Repository, packID restic.ID) pack.Blobs {
var blobs pack.Blobs
for pb := range repo.idx.Values() {
if pb.PackID().Equal(packID) {
blobs = append(blobs, pb.Blob)
}
}
blobs.Sort()
return blobs
}
internal/repository/pack/blob.go
+32
View File
@@ -0,0 +1,32 @@
package pack
import (
"fmt"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/restic"
)
// Blob is one part of a file or a tree with pack layout information.
type Blob struct {
restic.BlobHandle
Length uint
Offset uint
UncompressedLength uint
}
func (b Blob) String() string {
return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v, uncompressed length %v>",
b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength)
}
func (b Blob) DataLength() uint {
if b.UncompressedLength != 0 {
return b.UncompressedLength
}
return uint(crypto.PlaintextLength(int(b.Length)))
}
func (b Blob) IsCompressed() bool {
return b.UncompressedLength != 0
}
internal/repository/pack/blobs.go
+15
View File
@@ -0,0 +1,15 @@
package pack
import (
"cmp"
"slices"
)
// Blobs is a list of blobs with pack layout information (offset, length, ...).
type Blobs []Blob
func (b Blobs) Sort() {
slices.SortFunc(b, func(a, b Blob) int {
return cmp.Compare(a.Offset, b.Offset)
})
}
internal/repository/pack/blobs_test.go
+24
View File
@@ -0,0 +1,24 @@
package pack
import (
"testing"
rtest "github.com/restic/restic/internal/test"
)
func TestBlobsSort(t *testing.T) {
blobs := Blobs{
{Offset: 100},
{Offset: 0},
{Offset: 50},
}
blobs.Sort()
rtest.Equals(t, uint(0), blobs[0].Offset)
rtest.Equals(t, uint(50), blobs[1].Offset)
rtest.Equals(t, uint(100), blobs[2].Offset)
}
func TestBlobsSortNilSlice(t *testing.T) {
var blobs Blobs
blobs.Sort()
}
internal/repository/pack/pack.go
+19 -18
View File
@@ -21,7 +21,7 @@ var ErrBroken = errors.New("packer cannot be used after a write error")
// Packer is used to create a new Pack.
type Packer struct {
blobs restic.Blobs
blobs []Blob
bytes uint
k *crypto.Key
@@ -56,7 +56,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL
return n, p.err
}
c := restic.Blob{
c := Blob{
BlobHandle: restic.BlobHandle{Type: t, ID: id},
Length: uint(n),
Offset: p.bytes,
@@ -65,7 +65,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL
p.bytes += uint(n)
p.blobs = append(p.blobs, c)
return n + CalculateEntrySize(c), nil
return n + CalculateEntrySize(c.IsCompressed()), nil
}
var entrySize = uint(binary.Size(restic.BlobType(0)) + 2*headerLengthSize + len(restic.ID{}))
@@ -129,7 +129,7 @@ func (p *Packer) Finalize() error {
return nil
}
func verifyHeader(k *crypto.Key, header []byte, expected restic.Blobs) error {
func verifyHeader(k *crypto.Key, header []byte, expected []Blob) error {
// do not offer a way to skip the pack header verification, as pack headers are usually small enough
// to not result in a significant performance impact
@@ -157,7 +157,7 @@ func (p *Packer) HeaderOverhead() int {
}
// makeHeader constructs the header for p.
func makeHeader(blobs restic.Blobs) ([]byte, error) {
func makeHeader(blobs []Blob) ([]byte, error) {
buf := make([]byte, 0, len(blobs)*int(entrySize))
for _, b := range blobs {
@@ -232,7 +232,7 @@ func (p *Packer) HeaderFull() bool {
}
// Blobs returns the slice of blobs that have been written.
func (p *Packer) Blobs() restic.Blobs {
func (p *Packer) Blobs() Blobs {
p.m.Lock()
defer p.m.Unlock()
@@ -348,7 +348,7 @@ func (e InvalidFileError) Error() string {
// List returns the list of entries found in a pack file and the length of the
// header (including header size and crypto overhead)
func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrSize uint32, err error) {
func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries Blobs, hdrSize uint32, err error) {
buf, err := readHeader(rd, size)
if err != nil {
return nil, 0, err
@@ -367,7 +367,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS
}
// might over allocate a bit if all blobs have EntrySize but only by a few percent
entries = make(restic.Blobs, 0, uint(len(buf))/plainEntrySize)
entries = make(Blobs, 0, uint(len(buf))/plainEntrySize)
pos := uint(0)
for len(buf) > 0 {
@@ -385,7 +385,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS
return entries, hdrSize, nil
}
func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) {
func parseHeaderEntry(p []byte) (b Blob, size uint, err error) {
l := uint(len(p))
size = plainEntrySize
if l < plainEntrySize {
@@ -420,17 +420,17 @@ func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) {
return b, size, nil
}
func CalculateEntrySize(blob restic.Blob) int {
if blob.UncompressedLength != 0 {
func CalculateEntrySize(compressed bool) int {
if compressed {
return int(entrySize)
}
return int(plainEntrySize)
}
func CalculateHeaderSize(blobs restic.Blobs) int {
func CalculateHeaderSize(blobs Blobs) int {
size := headerSize
for _, blob := range blobs {
size += CalculateEntrySize(blob)
size += CalculateEntrySize(blob.IsCompressed())
}
return size
}
@@ -439,18 +439,19 @@ func CalculateHeaderSize(blobs restic.Blobs) int {
// If onlyHdr is set to true, only the size of the header is returned
// Note that this function only gives correct sizes, if there are no
// duplicates in the index.
func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) {
func Size(ctx context.Context, idx restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) {
packSize := make(map[restic.ID]int64)
err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) {
size, ok := packSize[blob.PackID]
err := idx.ListBlobs(ctx, func(blob restic.PackBlob) {
packID := blob.PackID()
size, ok := packSize[packID]
if !ok {
size = headerSize
}
if !onlyHdr {
size += int64(blob.Length)
size += int64(blob.CiphertextLength())
}
packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob))
packSize[packID] = size + int64(CalculateEntrySize(blob.IsCompressed()))
})
return packSize, err
internal/repository/pack/pack_internal_test.go
+1 -1
View File
@@ -182,7 +182,7 @@ func TestReadRecords(t *testing.T) {
func TestUnpackedVerification(t *testing.T) {
// create random keys
k := crypto.NewRandomKey()
blobs := restic.Blobs{
blobs := []Blob{
{
BlobHandle: restic.NewRandomBlobHandle(),
Length: 42,
internal/repository/pack/packedblob.go
+19
View File
@@ -0,0 +1,19 @@
package pack
import "github.com/restic/restic/internal/restic"
// PackedBlob is one index entry for a blob in a pack (may be duplicate across indexes).
type PackedBlob struct {
Pack restic.ID
Blob Blob
}
func (pb *PackedBlob) PackID() restic.ID { return pb.Pack }
func (pb *PackedBlob) Handle() restic.BlobHandle { return pb.Blob.BlobHandle }
func (pb *PackedBlob) CiphertextLength() uint { return pb.Blob.Length }
func (pb *PackedBlob) PlaintextLength() uint { return pb.Blob.DataLength() }
func (pb *PackedBlob) IsCompressed() bool { return pb.Blob.IsCompressed() }
internal/repository/prune.go
+22 -18
View File
@@ -142,11 +142,12 @@ func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsed
if len(plan.repackPacks) != 0 {
// when repacking, we do not want to keep blobs which are
// already contained in kept packs, so delete them from keepBlobs
err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
err := repo.ListBlobs(ctx, func(blob restic.PackBlob) {
packID := blob.PackID()
if plan.removePacks.Has(packID) || plan.repackPacks.Has(packID) {
return
}
keepBlobs.Delete(blob.BlobHandle)
keepBlobs.Delete(blob.Handle())
})
if err != nil {
return nil, err
@@ -179,8 +180,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
// iterate over all blobs in index to find out which blobs are duplicates
// The counter in usedBlobs describes how many instances of the blob exist in the repository index
// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
bh := blob.BlobHandle
err := idx.ListBlobs(ctx, func(blob restic.PackBlob) {
bh := blob.Handle()
count, ok := usedBlobs.Get(bh)
if ok {
if count < math.MaxUint8 {
@@ -229,22 +230,24 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
hasDuplicates := false
// iterate over all blobs in index to generate packInfo
err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
ip := indexPack[blob.PackID]
err = idx.ListBlobs(ctx, func(blob restic.PackBlob) {
packID := blob.PackID()
h := blob.Handle()
ip := indexPack[packID]
// Set blob type if not yet set
if ip.tpe == restic.NumBlobTypes {
ip.tpe = blob.Type
ip.tpe = h.Type
}
// mark mixed packs with "Invalid blob type"
if ip.tpe != blob.Type {
if ip.tpe != h.Type {
ip.tpe = restic.InvalidBlob
}
bh := blob.BlobHandle
size := uint64(blob.Length)
dupCount, _ := usedBlobs.Get(bh)
size := uint64(blob.CiphertextLength())
dupCount, _ := usedBlobs.Get(h)
switch {
case dupCount >= 2:
hasDuplicates = true
@@ -273,7 +276,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
ip.uncompressed = true
}
// update indexPack
indexPack[blob.PackID] = ip
indexPack[packID] = ip
})
if err != nil {
return nil, nil, err
@@ -287,8 +290,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
if hasDuplicates {
// iterate again over all blobs in index (this is pretty cheap, all in-mem)
err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) {
bh := blob.BlobHandle
err = idx.ListBlobs(ctx, func(blob restic.PackBlob) {
packID := blob.PackID()
bh := blob.Handle()
count, ok := usedBlobs.Get(bh)
// skip non-duplicate, aka. normal blobs
// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
@@ -296,8 +300,8 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
return
}
ip := indexPack[blob.PackID]
size := uint64(blob.Length)
ip := indexPack[packID]
size := uint64(blob.CiphertextLength())
switch {
case ip.usedBlobs > 0, ip.duplicateBlobs == ip.unusedBlobs, count == 0:
// other used blobs in pack, only duplicate blobs or "last" occurrence -> transition to used
@@ -325,7 +329,7 @@ func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *i
usedBlobs.Set(bh, count)
}
// update indexPack
indexPack[blob.PackID] = ip
indexPack[packID] = ip
})
if err != nil {
return nil, nil, err
internal/repository/repack.go
+9 -7
View File
@@ -7,6 +7,8 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/feature"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
@@ -30,7 +32,7 @@ type LogFunc func(msg string, args ...interface{})
// blobs have been processed.
func CopyBlobs(
ctx context.Context,
repo restic.Repository,
repo *Repository,
dstRepo restic.Repository,
dstUploader restic.BlobSaverWithAsync,
packs restic.IDSet,
@@ -55,7 +57,7 @@ func CopyBlobs(
func repack(
ctx context.Context,
repo restic.Repository,
repo *Repository,
dstRepo restic.Repository,
uploader restic.BlobSaverWithAsync,
packs restic.IDSet,
@@ -80,11 +82,11 @@ func repack(
}
var keepMutex sync.Mutex
downloadQueue := make(chan restic.PackBlobs)
downloadQueue := make(chan index.PackBlobs)
wg.Go(func() error {
defer close(downloadQueue)
for pbs := range repo.ListPacksFromIndex(wgCtx, packs) {
var packBlobs restic.Blobs
for pbs := range repo.listPacksFromIndex(wgCtx, packs) {
var packBlobs pack.Blobs
keepMutex.Lock()
// filter out unnecessary blobs
for _, entry := range pbs.Blobs {
@@ -96,7 +98,7 @@ func repack(
keepMutex.Unlock()
select {
case downloadQueue <- restic.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}:
case downloadQueue <- index.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}:
case <-wgCtx.Done():
return wgCtx.Err()
}
@@ -106,7 +108,7 @@ func repack(
worker := func() error {
for t := range downloadQueue {
err := repo.LoadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
err := repo.loadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
if err != nil {
// a required blob couldn't be retrieved
return err
internal/repository/repack_test.go
+26 -20
View File
@@ -86,13 +86,12 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
blobs := restic.NewBlobSet()
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
entries, err := repo.ListPack(context.TODO(), id, size)
handles, err := repo.ListPackHandles(context.TODO(), id, size)
if err != nil {
t.Fatalf("error listing pack %v: %v", id, err)
}
for _, entry := range entries {
h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
for _, h := range handles {
if blobs.Has(h) {
t.Errorf("ignoring duplicate blob %v", h)
return nil
@@ -100,9 +99,9 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
blobs.Insert(h)
if random.Float32() <= p {
list1.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type})
list1.Insert(h)
} else {
list2.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type})
list2.Insert(h)
}
}
return nil
@@ -142,14 +141,14 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
}
for _, pb := range list {
packs.Insert(pb.PackID)
packs.Insert(pb.PackID())
}
}
return packs
}
func repack(t *testing.T, repo restic.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) {
func repack(t *testing.T, repo *repository.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) {
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, nil, nil)
}))
@@ -222,8 +221,8 @@ func testRepack(t *testing.T, version uint) {
pb := list[0]
if removePacks.Has(pb.PackID) {
t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID)
if removePacks.Has(pb.PackID()) {
t.Errorf("lookup returned pack ID %v that should've been removed", pb.PackID())
}
}
@@ -238,21 +237,28 @@ func TestRepackCopy(t *testing.T) {
repository.TestAllVersions(t, testRepackCopy)
}
type oneConnectionRepo struct {
restic.Repository
// oneConnectionBackend limits concurrent backend operations to test repack with
// the minimum connection count required by CopyBlobs.
type oneConnectionBackend struct {
backend.Backend
}
func (r oneConnectionRepo) Connections() uint {
return 1
func (be *oneConnectionBackend) Properties() backend.Properties {
p := be.Backend.Properties()
p.Connections = 1
return p
}
func (be *oneConnectionBackend) Unwrap() backend.Backend {
return be.Backend
}
func testRepackCopy(t *testing.T, version uint) {
repo, _, _ := repository.TestRepositoryWithVersion(t, version)
dstRepo, _, _ := repository.TestRepositoryWithVersion(t, version)
// test with minimal possible connection count
repoWrapped := &oneConnectionRepo{repo}
dstRepoWrapped := &oneConnectionRepo{dstRepo}
repo, _ := repository.TestRepositoryWithBackend(t, &oneConnectionBackend{Backend: repository.TestBackend(t)}, version, repository.Options{})
dstRepo, _ := repository.TestRepositoryWithBackend(t, &oneConnectionBackend{Backend: repository.TestBackend(t)}, version, repository.Options{})
rtest.Equals(t, repo.Connections(), 1)
rtest.Equals(t, dstRepo.Connections(), 1)
seed := time.Now().UnixNano()
random := rand.New(rand.NewSource(seed))
@@ -265,8 +271,8 @@ func testRepackCopy(t *testing.T, version uint) {
_, keepBlobs := selectBlobs(t, random, repo, 0.2)
copyPacks := findPacksForBlobs(t, repo, keepBlobs)
rtest.OK(t, repoWrapped.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
return repository.CopyBlobs(ctx, repoWrapped, dstRepoWrapped, uploader, copyPacks, keepBlobs, nil, nil)
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
return repository.CopyBlobs(ctx, repo, dstRepo, uploader, copyPacks, keepBlobs, nil, nil)
}))
rebuildAndReloadIndex(t, dstRepo)
internal/repository/repair_pack.go
+7 -6
View File
@@ -6,6 +6,7 @@ import (
"io"
"slices"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
)
@@ -23,7 +24,7 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, ids) {
for b := range repo.listPacksFromIndex(ctx, ids) {
indexBlobs := b.Blobs
err := reuploadBlobsFromPack(ctx, repo, b.PackID, indexBlobs, printer, uploader)
if err != nil {
@@ -71,12 +72,12 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe
return nil
}
func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]restic.Blobs, error) {
packToBlobs := make(map[restic.ID]restic.Blobs)
func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]pack.Blobs, error) {
packToBlobs := make(map[restic.ID]pack.Blobs)
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
if ids.Has(id) {
blobs, err := repo.ListPack(ctx, id, size)
blobs, err := repo.listPack(ctx, id, size)
if err != nil {
return nil
}
@@ -90,8 +91,8 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
return packToBlobs, nil
}
func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs restic.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error {
err := repo.LoadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs pack.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error {
err := repo.loadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
if err != nil {
printer.E("failed to load blob %v: %v", blob.ID, err)
return nil
internal/repository/repair_pack_test.go
+10 -11
View File
@@ -16,8 +16,8 @@ import (
func listBlobs(repo restic.Repository) restic.BlobSet {
blobs := restic.NewBlobSet()
_ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
blobs.Insert(pb.BlobHandle)
_ = repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
blobs.Insert(pb.Handle())
})
return blobs
}
@@ -66,11 +66,10 @@ func testRepairBrokenPack(t *testing.T, version uint) {
// find blob that starts at offset 0
var damagedBlob restic.BlobHandle
for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) {
for _, blob := range blobs.Blobs {
if blob.Offset == 0 {
damagedBlob = blob.BlobHandle
}
for _, blob := range repository.BlobsInPack(repo, damagedID) {
if blob.Offset == 0 {
damagedBlob = blob.BlobHandle
break
}
}
@@ -89,11 +88,11 @@ func testRepairBrokenPack(t *testing.T, version uint) {
// all blobs in the file are broken
damagedBlobs := restic.NewBlobSet()
for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) {
for _, blob := range blobs.Blobs {
damagedBlobs.Insert(blob.BlobHandle)
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
if pb.PackID().Equal(damagedID) {
damagedBlobs.Insert(pb.Handle())
}
}
}))
return restic.NewIDSet(damagedID), damagedBlobs
},
}, {
internal/repository/repository.go
+72 -27
View File
@@ -213,7 +213,7 @@ type haver interface {
}
// sortCachedPacksFirst moves all cached pack files to the front of blobs.
func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) {
func sortCachedPacksFirst(cache haver, blobs []*pack.PackedBlob) {
if cache == nil {
return
}
@@ -224,10 +224,10 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) {
}
cached := blobs[:0]
noncached := make([]restic.PackedBlob, 0, len(blobs)/2)
noncached := make([]*pack.PackedBlob, 0, len(blobs)/2)
for _, blob := range blobs {
if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) {
if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID().String()}) {
cached = append(cached, blob)
continue
}
@@ -256,7 +256,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
if err != nil {
if r.cache != nil {
for _, blob := range blobs {
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()}
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()}
// ignore errors as there's not much we can do here
_ = r.cache.Forget(h)
}
@@ -267,28 +267,28 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
return buf, err
}
func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, buf []byte) ([]byte, error) {
func (r *Repository) loadBlob(ctx context.Context, blobs []*pack.PackedBlob, buf []byte) ([]byte, error) {
var lastError error
for _, blob := range blobs {
debug.Log("blob %v found: %v", blob.BlobHandle, blob)
debug.Log("blob %v found: %v", blob.Handle(), blob)
// load blob from pack
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()}
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()}
switch {
case cap(buf) < int(blob.Length):
buf = make([]byte, blob.Length)
case len(buf) != int(blob.Length):
buf = buf[:blob.Length]
case cap(buf) < int(blob.Blob.Length):
buf = make([]byte, blob.Blob.Length)
case len(buf) != int(blob.Blob.Length):
buf = buf[:blob.Blob.Length]
}
_, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf)
_, err := backend.ReadAt(ctx, r.be, h, int64(blob.Blob.Offset), buf)
if err != nil {
debug.Log("error loading blob %v: %v", blob, err)
lastError = err
continue
}
it := newPackBlobIterator(blob.PackID, newByteReader(buf), blob.Offset, restic.Blobs{blob.Blob}, r.key, r.getZstdDecoder())
it := newPackBlobIterator(blob.PackID(), newByteReader(buf), blob.Blob.Offset, pack.Blobs{blob.Blob}, r.key, r.getZstdDecoder())
pbv, err := it.Next()
if err == nil {
@@ -314,7 +314,7 @@ func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, bu
return nil, lastError
}
return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs))
return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].Handle(), len(blobs))
}
func (r *Repository) getZstdEncoder() *zstd.Encoder {
@@ -672,8 +672,13 @@ func (r *Repository) Connections() uint {
return r.be.Properties().Connections
}
func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackedBlob {
return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id})
func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob {
entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id})
out := make([]restic.PackBlob, len(entries))
for i, e := range entries {
out[i] = e
}
return out
}
// LookupBlobSize returns the size of blob id. Also returns pending blobs.
@@ -683,7 +688,7 @@ func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bo
// ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error {
func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) error {
for blob := range r.idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
@@ -693,7 +698,8 @@ func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob))
return nil
}
func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs {
// listPacksFromIndex returns index entries for the given packs, grouped by pack file.
func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan index.PackBlobs {
return r.idx.ListPacks(ctx, packs)
}
@@ -783,7 +789,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
worker := func() error {
for fi := range ch {
entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
entries, err := r.listPack(wgCtx, fi.ID, fi.Size)
if err != nil {
debug.Log("unable to list pack file %v", fi.ID.Str())
m.Lock()
@@ -964,8 +970,8 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
})
}
// ListPack returns the list of blobs saved in the pack id.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
// listPack returns blob entries from the pack file header including offsets.
func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (pack.Blobs, error) {
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
@@ -978,7 +984,20 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
// retry on error
entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
}
return entries, err
return pack.Blobs(entries), err
}
// ListPackHandles returns the blob handles stored in the pack file header.
func (r *Repository) ListPackHandles(ctx context.Context, id restic.ID, size int64) ([]restic.BlobHandle, error) {
blobs, err := r.listPack(ctx, id, size)
if err != nil {
return nil, err
}
handles := make([]restic.BlobHandle, len(blobs))
for i, blob := range blobs {
handles[i] = blob.BlobHandle
}
return handles, nil
}
// Delete calls backend.Delete() if implemented, and returns an error
@@ -1054,11 +1073,37 @@ const maxUnusedRange = 1 * 1024 * 1024
// handleBlobFn is called at most once for each blob. If the callback returns an error,
// then LoadBlobsFromPack will abort and not retry it. The buf passed to the callback is only valid within
// this specific call. The callback must not keep a reference to buf.
func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
blobs, err := r.blobsInPack(packID, handles)
if err != nil {
return err
}
return r.loadBlobsFromPack(ctx, packID, blobs, handleBlobFn)
}
func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (pack.Blobs, error) {
blobs := make(pack.Blobs, 0, len(handles))
for _, h := range handles {
found := false
for _, pb := range r.idx.Lookup(h) {
if pb.PackID().Equal(packID) {
blobs = append(blobs, pb.Blob)
found = true
break
}
}
if !found {
return nil, errors.Errorf("blob %v not found in pack %v", h, packID)
}
}
return blobs, nil
}
func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
return streamPack(ctx, r.be.Load, r.LoadBlob, r.getZstdDecoder(), r.key, packID, blobs, handleBlobFn)
}
func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
if len(blobs) == 0 {
// nothing to do
return nil
@@ -1101,7 +1146,7 @@ func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn
return streamPackPart(ctx, beLoad, loadBlobFn, dec, key, packID, blobs[lowerIdx:], handleBlobFn)
}
func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: blobs[0].Type.IsMetadata()}
dataStart := blobs[0].Offset
@@ -1211,7 +1256,7 @@ type packBlobIterator struct {
rd discardReader
currentOffset uint
blobs restic.Blobs
blobs pack.Blobs
key *crypto.Key
dec *zstd.Decoder
@@ -1227,7 +1272,7 @@ type packBlobValue struct {
var errPackEOF = errors.New("reached EOF of pack file")
func newPackBlobIterator(packID restic.ID, rd discardReader, currentOffset uint,
blobs restic.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator {
blobs pack.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator {
return &packBlobIterator{
packID: packID,
rd: rd,
internal/repository/repository_internal_test.go
+19 -18
View File
@@ -17,6 +17,7 @@ import (
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -27,7 +28,7 @@ func (c mapcache) Has(h backend.Handle) bool { return c[h] }
func TestSortCachedPacksFirst(t *testing.T) {
var (
blobs, sorted [100]restic.PackedBlob
blobs, sorted [100]*pack.PackedBlob
cache = make(mapcache)
r = rand.New(rand.NewSource(1261))
@@ -36,7 +37,7 @@ func TestSortCachedPacksFirst(t *testing.T) {
for i := 0; i < len(blobs); i++ {
var id restic.ID
r.Read(id[:])
blobs[i] = restic.PackedBlob{PackID: id}
blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}}
if i%3 == 0 {
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
@@ -46,8 +47,8 @@ func TestSortCachedPacksFirst(t *testing.T) {
copy(sorted[:], blobs[:])
sort.SliceStable(sorted[:], func(i, j int) bool {
hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()}
hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()}
hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID().String()}
hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID().String()}
return cache.Has(hi) && !cache.Has(hj)
})
@@ -59,7 +60,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
const nblobs = 512 // Corresponds to a file of ca. 2GB.
var (
blobs [nblobs]restic.PackedBlob
blobs [nblobs]*pack.PackedBlob
cache = make(mapcache)
r = rand.New(rand.NewSource(1261))
)
@@ -67,7 +68,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
for i := 0; i < nblobs; i++ {
var id restic.ID
r.Read(id[:])
blobs[i] = restic.PackedBlob{PackID: id}
blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}}
if i%3 == 0 {
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
@@ -75,7 +76,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
}
}
var cpy [nblobs]restic.PackedBlob
var cpy [nblobs]*pack.PackedBlob
b.ReportAllocs()
b.ResetTimer()
@@ -96,7 +97,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
idx := index.NewIndex()
for i := 0; i < 5000; i++ {
idx.StorePack(restic.NewRandomID(), restic.Blobs{
idx.StorePack(restic.NewRandomID(), pack.Blobs{
{
BlobHandle: restic.NewRandomBlobHandle(),
Length: 1234,
@@ -133,7 +134,7 @@ func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*
}
// buildPackfileWithoutHeader returns a manually built pack file without a header.
func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs restic.Blobs, packfile []byte) {
func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs pack.Blobs, packfile []byte) {
opts := []zstd.EOption{
// Set the compression level configured.
zstd.WithEncoderLevel(zstd.SpeedDefault),
@@ -173,7 +174,7 @@ func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool)
ciphertextLength := after - before
blobs = append(blobs, restic.Blob{
blobs = append(blobs, pack.Blob{
BlobHandle: restic.BlobHandle{
Type: restic.DataBlob,
ID: id,
@@ -280,19 +281,19 @@ func testStreamPack(t *testing.T, version uint) {
// first, test regular usage
t.Run("regular", func(t *testing.T) {
tests := []struct {
blobs restic.Blobs
blobs pack.Blobs
calls int
shortFirstLoad bool
}{
{packfileBlobs[1:2], 1, false},
{packfileBlobs[2:5], 1, false},
{packfileBlobs[2:8], 1, false},
{restic.Blobs{
{pack.Blobs{
packfileBlobs[0],
packfileBlobs[4],
packfileBlobs[2],
}, 1, false},
{restic.Blobs{
{pack.Blobs{
packfileBlobs[0],
packfileBlobs[len(packfileBlobs)-1],
}, 2, false},
@@ -341,12 +342,12 @@ func testStreamPack(t *testing.T, version uint) {
// next, test invalid uses, which should return an error
t.Run("invalid", func(t *testing.T) {
tests := []struct {
blobs restic.Blobs
blobs pack.Blobs
err string
}{
{
// pass one blob several times
blobs: restic.Blobs{
blobs: pack.Blobs{
packfileBlobs[3],
packfileBlobs[8],
packfileBlobs[3],
@@ -357,7 +358,7 @@ func testStreamPack(t *testing.T, version uint) {
{
// pass something that's not a valid blob in the current pack file
blobs: restic.Blobs{
blobs: pack.Blobs{
{
Offset: 123,
Length: 20000,
@@ -368,7 +369,7 @@ func testStreamPack(t *testing.T, version uint) {
{
// pass a blob that's too small
blobs: restic.Blobs{
blobs: pack.Blobs{
{
Offset: 123,
Length: 10,
@@ -523,7 +524,7 @@ func TestStreamPackFallback(t *testing.T) {
plaintext := rtest.Random(800, 42)
blobID := restic.Hash(plaintext)
blobs := restic.Blobs{
blobs := pack.Blobs{
{
Length: uint(crypto.CiphertextLength(len(plaintext))),
Offset: 0,
internal/repository/repository_test.go
+9 -8
View File
@@ -233,7 +233,7 @@ func TestLoadBlobBroken(t *testing.T) {
data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil)
rtest.OK(t, err)
rtest.Assert(t, bytes.Equal(buf, data), "data mismatch")
pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID
pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID()
rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached")
}
@@ -422,11 +422,12 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
rtest.OK(t, err)
for pb := range idx.Values() {
if _, ok := packEntries[pb.PackID]; !ok {
packEntries[pb.PackID] = make(map[restic.ID]struct{})
packID := pb.PackID()
if _, ok := packEntries[packID]; !ok {
packEntries[packID] = make(map[restic.ID]struct{})
}
packEntries[pb.PackID][id] = struct{}{}
packEntries[packID][id] = struct{}{}
}
return nil
})
@@ -467,7 +468,7 @@ func TestListPack(t *testing.T) {
repo.UseCache(c, t.Logf)
// Forcibly cache pack file
packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID
packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID()
rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil }))
// Get size to list pack
@@ -479,11 +480,11 @@ func TestListPack(t *testing.T) {
return nil
}))
blobs, err := repo.ListPack(context.TODO(), packID, size)
handles, err := repo.ListPackHandles(context.TODO(), packID, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
rtest.Assert(t, len(handles) == 1 && handles[0].ID == id, "unexpected blobs in pack: %v", handles)
rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as ListPack does not set IsMetadata in the backend.Handle")
rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as listPack does not set IsMetadata in the backend.Handle")
}
func TestNoDoubleInit(t *testing.T) {
internal/restic/blob.go
+11 -39
View File
@@ -1,50 +1,22 @@
package restic
import (
"cmp"
"fmt"
"slices"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
)
// Blob is one part of a file or a tree.
type Blob struct {
BlobHandle
Length uint
Offset uint
UncompressedLength uint
}
func (b Blob) String() string {
return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v, uncompressed length %v>",
b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength)
}
func (b Blob) DataLength() uint {
if b.UncompressedLength != 0 {
return b.UncompressedLength
}
return uint(crypto.PlaintextLength(int(b.Length)))
}
func (b Blob) IsCompressed() bool {
return b.UncompressedLength != 0
}
type Blobs []Blob
func (b Blobs) Sort() {
slices.SortFunc(b, func(a, b Blob) int {
return cmp.Compare(a.Offset, b.Offset)
})
}
// PackedBlob is a blob stored within a file.
type PackedBlob struct {
Blob
PackID ID
// PackBlob is one index entry for a blob in a pack file.
// The interface intentionally omits the offset at which a blob is stored in the pack.
// This ensures that pack file internals are not leaked.
type PackBlob interface {
PackID() ID
Handle() BlobHandle
// CiphertextLength is the encrypted size stored in the pack.
CiphertextLength() uint
// PlaintextLength is the size after decryption/decompression.
PlaintextLength() uint
IsCompressed() bool
}
// BlobHandle identifies a blob of a given type.
internal/restic/blob_test.go
-19
View File
@@ -3,8 +3,6 @@ package restic
import (
"encoding/json"
"testing"
rtest "github.com/restic/restic/internal/test"
)
var blobTypeJSON = []struct {
@@ -41,20 +39,3 @@ func TestBlobTypeJSON(t *testing.T) {
}
}
}
func TestBlobsSort(t *testing.T) {
blobs := Blobs{
{Offset: 100},
{Offset: 0},
{Offset: 50},
}
blobs.Sort()
rtest.Equals(t, uint(0), blobs[0].Offset)
rtest.Equals(t, uint(50), blobs[1].Offset)
rtest.Equals(t, uint(100), blobs[2].Offset)
}
func TestBlobsSortNilSlice(t *testing.T) {
var blobs Blobs
blobs.Sort()
}
internal/restic/repository.go
+6 -12
View File
@@ -24,19 +24,18 @@ type Repository interface {
LoadIndex(ctx context.Context, p TerminalCounterFactory) error
LookupBlob(t BlobType, id ID) []PackedBlob
LookupBlob(t BlobType, id ID) []PackBlob
LookupBlobSize(t BlobType, id ID) (size uint, exists bool)
NewAssociatedBlobSet() AssociatedBlobSet
// ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
// ListPack returns the list of blobs saved in the pack id.
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
ListBlobs(ctx context.Context, fn func(PackBlob)) error
// ListPackHandles returns the blob handles stored in the pack file header.
ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error)
LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error
LoadBlobsFromPack(ctx context.Context, packID ID, blobs []BlobHandle, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error
// WithUploader starts the necessary workers to upload new blobs. Once the callback returns,
// the workers are stopped and the index is written to the repository. The callback must use
@@ -125,11 +124,6 @@ type SaverRemoverUnpacked[FT FileTypes] interface {
RemoverUnpacked[FT]
}
type PackBlobs struct {
PackID ID
Blobs Blobs
}
type TerminalCounterFactory interface {
// NewCounterTerminalOnly returns a new progress counter that is only shown if stdout points to a
// terminal. It is not shown if --quiet or --json is specified.
@@ -153,7 +147,7 @@ type Unpacked[FT FileTypes] interface {
}
type ListBlobser interface {
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
ListBlobs(ctx context.Context, fn func(PackBlob)) error
}
type BlobLoader interface {
internal/restorer/filerestorer.go
+20 -19
View File
@@ -42,12 +42,12 @@ type packInfo struct {
files map[*fileInfo]struct{} // set of files that use blobs from this pack
}
type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error
type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error
type startWarmupFn func(context.Context, restic.IDSet) (restic.WarmupJob, error)
// fileRestorer restores set of files
type fileRestorer struct {
idx func(restic.BlobType, restic.ID) []restic.PackedBlob
idx func(restic.BlobType, restic.ID) []restic.PackBlob
blobsLoader blobsLoaderFn
startWarmup startWarmupFn
@@ -68,7 +68,7 @@ type fileRestorer struct {
func newFileRestorer(dst string,
blobsLoader blobsLoaderFn,
idx func(restic.BlobType, restic.ID) []restic.PackedBlob,
idx func(restic.BlobType, restic.ID) []restic.PackBlob,
connections uint,
sparse bool,
allowRecursiveDelete bool,
@@ -102,7 +102,7 @@ func (r *fileRestorer) targetPath(location string) string {
return filepath.Join(r.dst, location)
}
func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int, fileOffset int64)) error {
func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(blob restic.PackBlob, idx int, fileOffset int64)) error {
if len(blobIDs) == 0 {
return nil
}
@@ -114,8 +114,8 @@ func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID
return errors.Errorf("Unknown blob %s", blobID.String())
}
pb := packs[0]
fn(pb.PackID, pb.Blob, i, fileOffset)
fileOffset += int64(pb.DataLength())
fn(pb, i, fileOffset)
fileOffset += int64(pb.PlaintextLength())
}
return nil
@@ -143,14 +143,15 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
file.blobs = packsMap
}
restoredBlobs := false
err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) {
err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) {
packID := blob.PackID()
if !file.state.HasMatchingBlob(idx) {
if largeFile {
packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset})
packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.Handle().ID, offset: fileOffset})
}
restoredBlobs = true
} else {
r.reportBlobProgress(file, uint64(blob.DataLength()))
r.reportBlobProgress(file, uint64(blob.PlaintextLength()))
// completely ignore blob
return
}
@@ -164,7 +165,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
packOrder = append(packOrder, packID)
}
pack.files[file] = struct{}{}
if blob.ID.Equal(r.zeroChunk) {
if blob.Handle().ID.Equal(r.zeroChunk) {
file.sparse = r.sparse
}
})
@@ -261,14 +262,14 @@ func (r *fileRestorer) truncateFileToSize(location string, size int64) error {
type blobToFileOffsetsMapping map[restic.ID]struct {
files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file
blob restic.Blob
blob restic.BlobHandle
}
func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
// calculate blob->[]files->[]offsets mappings
blobs := make(blobToFileOffsetsMapping)
for file := range pack.files {
addBlob := func(blob restic.Blob, fileOffset int64) {
addBlob := func(blob restic.BlobHandle, fileOffset int64) {
blobInfo, ok := blobs[blob.ID]
if !ok {
blobInfo.files = make(map[*fileInfo][]int64)
@@ -278,9 +279,9 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
blobInfo.files[file] = append(blobInfo.files[file], fileOffset)
}
if fileBlobs, ok := file.blobs.(restic.IDs); ok {
err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) {
if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) {
addBlob(blob, fileOffset)
err := r.forEachBlob(fileBlobs, func(blob restic.PackBlob, idx int, fileOffset int64) {
if blob.PackID().Equal(pack.id) && !file.state.HasMatchingBlob(idx) {
addBlob(blob.Handle(), fileOffset)
}
})
if err != nil {
@@ -291,8 +292,8 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
for _, blob := range packsMap[pack.id] {
idxPacks := r.idx(restic.DataBlob, blob.id)
for _, idxPack := range idxPacks {
if idxPack.PackID.Equal(pack.id) {
addBlob(idxPack.Blob, blob.offset)
if idxPack.PackID().Equal(pack.id) {
addBlob(idxPack.Handle(), blob.offset)
break
}
}
@@ -324,7 +325,7 @@ func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlob
// only report error for not yet processed blobs
affectedFiles := make(map[*fileInfo]struct{})
for _, entry := range blobs {
if processedBlobs.Has(entry.blob.BlobHandle) {
if processedBlobs.Has(entry.blob) {
continue
}
for file := range entry.files {
@@ -343,7 +344,7 @@ func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlob
func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID,
blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error {
blobList := make(restic.Blobs, 0, len(blobs))
blobList := make([]restic.BlobHandle, 0, len(blobs))
for _, entry := range blobs {
blobList = append(blobList, entry.blob)
}
internal/restorer/filerestorer_test.go
+69 -31
View File
@@ -2,9 +2,11 @@ package restorer
import (
"bytes"
"cmp"
"context"
"fmt"
"os"
"slices"
"testing"
"github.com/restic/restic/internal/errors"
@@ -28,11 +30,32 @@ type TestWarmupJob struct {
waitCalled bool
}
type testPackBlob struct {
packID restic.ID
handle restic.BlobHandle
offset uint
ciphertext uint
plaintext uint
compressed bool
}
var _ restic.PackBlob = (*testPackBlob)(nil)
func (pb *testPackBlob) PackID() restic.ID { return pb.packID }
func (pb *testPackBlob) Handle() restic.BlobHandle { return pb.handle }
func (pb *testPackBlob) CiphertextLength() uint { return pb.ciphertext }
func (pb *testPackBlob) PlaintextLength() uint { return pb.plaintext }
func (pb *testPackBlob) IsCompressed() bool { return pb.compressed }
type TestRepo struct {
packsIDToData map[restic.ID][]byte
// blobs and files
blobs map[restic.ID][]restic.PackedBlob
blobs map[restic.ID][]restic.PackBlob
files []*fileInfo
filesPathToContent map[string]string
@@ -42,7 +65,7 @@ type TestRepo struct {
loader blobsLoaderFn
}
func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackedBlob {
func (i *TestRepo) Lookup(_ restic.BlobType, id restic.ID) []restic.PackBlob {
packs := i.blobs[id]
return packs
}
@@ -67,10 +90,16 @@ func (job *TestWarmupJob) Wait(_ context.Context) error {
}
func newTestRepo(content []TestFile) *TestRepo {
type packBlobLayout struct {
offset uint
ciphertext uint
plaintext uint
compressed bool
}
type Pack struct {
name string
data []byte
blobs map[restic.ID]restic.Blob
blobs map[restic.ID]packBlobLayout
}
packs := make(map[string]Pack)
filesPathToContent := make(map[string]string)
@@ -84,21 +113,19 @@ func newTestRepo(content []TestFile) *TestRepo {
var pack Pack
var found bool
if pack, found = packs[blob.pack]; !found {
pack = Pack{name: blob.pack, blobs: make(map[restic.ID]restic.Blob)}
pack = Pack{name: blob.pack, blobs: make(map[restic.ID]packBlobLayout)}
}
// calculate blob id and add to the pack as necessary
blobID := restic.Hash([]byte(blob.data))
if _, found := pack.blobs[blobID]; !found {
blobData := []byte(blob.data)
pack.blobs[blobID] = restic.Blob{
BlobHandle: restic.BlobHandle{
Type: restic.DataBlob,
ID: blobID,
},
Length: uint(len(blobData)),
UncompressedLength: uint(len(blobData)),
Offset: uint(len(pack.data)),
n := uint(len(blobData))
pack.blobs[blobID] = packBlobLayout{
offset: uint(len(pack.data)),
ciphertext: n,
plaintext: n,
compressed: true,
}
pack.data = append(pack.data, blobData...)
}
@@ -108,14 +135,19 @@ func newTestRepo(content []TestFile) *TestRepo {
filesPathToContent[file.name] = content
}
blobs := make(map[restic.ID][]restic.PackedBlob)
blobs := make(map[restic.ID][]restic.PackBlob)
packsIDToData := make(map[restic.ID][]byte)
for _, pack := range packs {
packID := restic.Hash(pack.data)
packsIDToData[packID] = pack.data
for blobID, blob := range pack.blobs {
blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID})
for blobID, layout := range pack.blobs {
blobs[blobID] = append(blobs[blobID], &testPackBlob{
packID: packID,
handle: restic.BlobHandle{Type: restic.DataBlob, ID: blobID},
offset: layout.offset, ciphertext: layout.ciphertext,
plaintext: layout.plaintext, compressed: layout.compressed,
})
}
}
@@ -135,24 +167,30 @@ func newTestRepo(content []TestFile) *TestRepo {
filesPathToContent: filesPathToContent,
warmupJobs: []*TestWarmupJob{},
}
repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
blobs = append(restic.Blobs{}, blobs...)
blobs.Sort()
for _, blob := range blobs {
repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
entries := make([]*testPackBlob, 0, len(handles))
for _, h := range handles {
found := false
for _, e := range repo.blobs[blob.ID] {
if packID == e.PackID {
for _, e := range repo.blobs[h.ID] {
if packID == e.PackID() {
entries = append(entries, e.(*testPackBlob))
found = true
buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length]
err := handleBlobFn(e.BlobHandle, buf, nil)
if err != nil {
return err
}
break
}
}
if !found {
return fmt.Errorf("missing blob: %v", blob)
return fmt.Errorf("missing blob: %v", h)
}
}
slices.SortFunc(entries, func(a, b *testPackBlob) int {
return cmp.Compare(a.offset, b.offset)
})
for _, e := range entries {
buf := repo.packsIDToData[packID][e.offset : e.offset+e.ciphertext]
err := handleBlobFn(e.handle, buf, nil)
if err != nil {
return err
}
}
return nil
@@ -313,7 +351,7 @@ func TestErrorRestoreFiles(t *testing.T) {
loadError := errors.New("load error")
// loader always returns an error
repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
return loadError
}
@@ -346,9 +384,9 @@ func TestFatalDownloadError(t *testing.T) {
repo := newTestRepo(content)
loader := repo.loader
repo.loader = func(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
repo.loader = func(ctx context.Context, packID restic.ID, handles []restic.BlobHandle, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
ctr := 0
return loader(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
return loader(ctx, packID, handles, func(blob restic.BlobHandle, buf []byte, err error) error {
if ctr < 2 {
ctr++
return handleBlobFn(blob, buf, err)