diff --git a/internal/repository/checker.go b/internal/repository/checker.go index 0e44ef80a..351dbc247 100644 --- a/internal/repository/checker.go +++ b/internal/repository/checker.go @@ -129,10 +129,11 @@ func (c *Checker) LoadIndex(ctx context.Context, p restic.TerminalCounterFactory } cnt++ - if _, ok := packToIndex[blob.PackID]; !ok { - packToIndex[blob.PackID] = restic.NewIDSet() + packID := blob.PackID() + if _, ok := packToIndex[packID]; !ok { + packToIndex[packID] = restic.NewIDSet() } - packToIndex[blob.PackID].Insert(id) + packToIndex[packID].Insert(id) } for pbs := range idx.EachByPack(ctx, restic.NewIDSet()) { @@ -259,7 +260,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID type checkTask struct { id restic.ID size int64 - blobs restic.Blobs + blobs pack.Blobs } ch := make(chan checkTask) @@ -329,7 +330,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID } // checkPack reads a pack and checks the integrity of all blobs. -func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { +func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec) if err != nil { if r.cache != nil { @@ -348,7 +349,7 @@ func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Bl return err } -func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { +func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { type partialReadError struct { error @@ -465,7 +466,7 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest // Check if blob is contained in index and position is correct idxHas := false for _, pb := range r.idx.Lookup(blob.BlobHandle) { - if pb.PackID.Equal(id) && pb.Blob == blob { + if pb.PackID().Equal(id) && pb.Blob == blob { idxHas = true break } diff --git a/internal/repository/debug.go b/internal/repository/debug.go index 8bf25a715..dc0a81ce2 100644 --- a/internal/repository/debug.go +++ b/internal/repository/debug.go @@ -123,7 +123,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami checkPackSize(b.Blobs, len(buf), printer) - err = loadBlobs(ctx, opts, repo, id, b.Blobs, printer) + err := loadBlobs(ctx, opts, repo, id, b.Blobs, printer) if err != nil { printer.E("error: %v", err) } else { @@ -146,7 +146,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami return nil } -func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { +func checkPackSize(blobs pack.Blobs, fileSize int, printer progress.Printer) { // track current size and offset var size, offset uint64 @@ -284,7 +284,7 @@ func decryptUnsigned(k *crypto.Key, buf []byte) []byte { return out } -func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { +func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list pack.Blobs, printer progress.Printer) error { dec, err := zstd.NewReader(nil) if err != nil { panic(err) diff --git a/internal/repository/index/associated_data.go b/internal/repository/index/associated_data.go index 1267bdc4f..cd66f5c4e 100644 --- a/internal/repository/index/associated_data.go +++ b/internal/repository/index/associated_data.go @@ -159,14 +159,15 @@ func (a *AssociatedSet[T]) All() iter.Seq2[restic.BlobHandle, T] { } for pb := range a.idx.Values() { - if _, ok := a.overflow[pb.BlobHandle]; ok { + bh := pb.Handle() + if _, ok := a.overflow[bh]; ok { // already reported via overflow set continue } - val, known := a.Get(pb.BlobHandle) + val, known := a.Get(bh) if known { - if !yield(pb.BlobHandle, val) { + if !yield(bh, val) { return } } diff --git a/internal/repository/index/associated_data_test.go b/internal/repository/index/associated_data_test.go index 07e0a3d58..413f60990 100644 --- a/internal/repository/index/associated_data_test.go +++ b/internal/repository/index/associated_data_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -19,17 +20,17 @@ func (n *noopSaver) SaveUnpacked(_ context.Context, _ restic.FileType, buf []byt return restic.Hash(buf), nil } -func makeFakePackedBlob() (restic.BlobHandle, restic.PackedBlob) { +func makeFakePackedBlob() (restic.BlobHandle, *pack.PackedBlob) { bh := restic.NewRandomBlobHandle() - blob := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bh, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - return bh, blob + return bh, pb } func list(bs *AssociatedSet[uint8]) restic.BlobHandles { @@ -40,7 +41,7 @@ func TestAssociatedSet(t *testing.T) { bh, blob := makeFakePackedBlob() mi := NewMasterIndex() - test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) @@ -123,14 +124,14 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) { _, blob := makeFakePackedBlob() mi := NewMasterIndex() - test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) bs := NewAssociatedSet[uint8](mi) // add new blobs to index after building the set of, blob2 := makeFakePackedBlob() - test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, &noopSaver{})) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, &noopSaver{})) test.OK(t, mi.Flush(context.TODO(), &noopSaver{})) // non-existent @@ -167,10 +168,10 @@ func TestAssociatedSetIntersectAndSub(t *testing.T) { bh3, blob3 := makeFakePackedBlob() bh4, blob4 := makeFakePackedBlob() - test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, restic.Blobs{blob3.Blob}, saver)) - test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, restic.Blobs{blob4.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob3.PackID(), pack.Blobs{blob3.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob4.PackID(), pack.Blobs{blob4.Blob}, saver)) test.OK(t, mi.Flush(context.TODO(), saver)) t.Run("Intersect", func(t *testing.T) { diff --git a/internal/repository/index/index.go b/internal/repository/index/index.go index abd8eda54..961b5aa89 100644 --- a/internal/repository/index/index.go +++ b/internal/repository/index/index.go @@ -73,7 +73,7 @@ func (idx *Index) addToPacks(id restic.ID) int { return len(idx.packs) - 1 } -func (idx *Index) store(packIndex int, blob restic.Blob) { +func (idx *Index) store(packIndex int, blob pack.Blob) { // assert that offset and length fit into uint32! if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 { panic("offset or length does not fit in uint32. You have packs > 4GB!") @@ -146,7 +146,7 @@ func (idx *Index) Preallocate(t restic.BlobType, numEntries int) { // StorePack remembers the ids of all blobs of a given pack // in the index -func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) { +func (idx *Index) StorePack(id restic.ID, blobs pack.Blobs) { idx.m.Lock() defer idx.m.Unlock() @@ -162,23 +162,24 @@ func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) { } } -func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) restic.PackedBlob { - return restic.PackedBlob{ - Blob: restic.Blob{ +func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) *pack.PackedBlob { + return &pack.PackedBlob{ + Pack: idx.packs[e.packIndex], + Blob: pack.Blob{ BlobHandle: restic.BlobHandle{ ID: e.id, - Type: t}, + Type: t, + }, Length: uint(e.length), Offset: uint(e.offset), UncompressedLength: uint(e.uncompressedLength), }, - PackID: idx.packs[e.packIndex], } } // Lookup queries the index for the blob ID and returns all entries including -// duplicates. Adds found entries to blobs and returns the result. -func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic.PackedBlob { +// duplicates. Adds found entries to pbs and returns the result. +func (idx *Index) Lookup(bh restic.BlobHandle, pbs []*pack.PackedBlob) []*pack.PackedBlob { idx.m.RLock() defer idx.m.RUnlock() @@ -215,8 +216,8 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found // Values returns an iterator over all blobs known to the index. This blocks any // modification of the index. -func (idx *Index) Values() iter.Seq[restic.PackedBlob] { - return func(yield func(restic.PackedBlob) bool) { +func (idx *Index) Values() iter.Seq[*pack.PackedBlob] { + return func(yield func(*pack.PackedBlob) bool) { idx.m.RLock() defer idx.m.RUnlock() @@ -231,17 +232,23 @@ func (idx *Index) Values() iter.Seq[restic.PackedBlob] { } } -// EachByPack returns a channel that yields all blobs known to the index +// PackBlobs lists all blobs contained in a pack file according to the index. +type PackBlobs struct { + PackID restic.ID + Blobs pack.Blobs +} + +// EachByPack returns a channel that yields all blobs known to the index, // grouped by packID but ignoring blobs with a packID in packPlacklist for // finalized indexes. // This filtering is used when rebuilding the index where we need to ignore packs // from the finalized index which have been re-read into a non-finalized index. // When the context is cancelled, the background goroutine // terminates. This blocks any modification of the index. -func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan restic.PackBlobs { +func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan PackBlobs { idx.m.RLock() - ch := make(chan restic.PackBlobs) + ch := make(chan PackBlobs) go func() { defer idx.m.RUnlock() @@ -262,7 +269,7 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <- } for packID, packByType := range byPack { - var result restic.PackBlobs + var result PackBlobs result.PackID = packID for typ, p := range packByType { for _, e := range p { @@ -482,7 +489,7 @@ func (idx *Index) merge(idx2 *Index) error { for e := range m.valuesWithID(e2.id) { b := idx.toPackedBlob(e, restic.BlobType(typ)) b2 := idx2.toPackedBlob(e2, restic.BlobType(typ)) - if b == b2 { + if b.Blob == b2.Blob && b.PackID() == b2.PackID() { found = true break } @@ -519,7 +526,7 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, err error) { packID := idx.addToPacks(p.ID) for _, blob := range p.Blobs { - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: blob.Type, ID: blob.ID}, @@ -550,7 +557,7 @@ func (idx *Index) Len(t restic.BlobType) uint { return idx.byType[t].len() } -func PackBlobsHash(pbs restic.PackBlobs) restic.ID { +func PackBlobsHash(pbs PackBlobs) restic.ID { h := sha256.New() h.Write(pbs.PackID[:]) diff --git a/internal/repository/index/index_internal_test.go b/internal/repository/index/index_internal_test.go index 67591ef8a..94a871ed3 100644 --- a/internal/repository/index/index_internal_test.go +++ b/internal/repository/index/index_internal_test.go @@ -14,7 +14,7 @@ func TestIndexOversized(t *testing.T) { // Add blobs up to indexMaxBlobs + pack.MaxHeaderEntries - 1 packID := idx.addToPacks(restic.NewRandomID()) for i := uint(0); i < indexMaxBlobs+pack.MaxHeaderEntries-1; i++ { - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), @@ -27,7 +27,7 @@ func TestIndexOversized(t *testing.T) { rtest.Assert(t, !Oversized(idx), "index should not be considered oversized") // Add one more blob to exceed the limit - idx.store(packID, restic.Blob{ + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), diff --git a/internal/repository/index/index_test.go b/internal/repository/index/index_test.go index 2a53072df..d5a779da3 100644 --- a/internal/repository/index/index_test.go +++ b/internal/repository/index/index_test.go @@ -9,19 +9,20 @@ import ( "testing" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func TestIndexSerialize(t *testing.T) { - tests := []restic.PackedBlob{} + tests := []*pack.PackedBlob{} idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 20; j++ { @@ -31,14 +32,14 @@ func TestIndexSerialize(t *testing.T) { // test a mix of compressed and uncompressed packs uncompressedLength = 2 * length } - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, UncompressedLength: uncompressedLength, }, - PackID: packID, } blobs = append(blobs, pb.Blob) tests = append(tests, pb) @@ -64,17 +65,17 @@ func TestIndexSerialize(t *testing.T) { rtest.OK(t, err) for _, testBlob := range tests { - list := idx.Lookup(testBlob.BlobHandle, nil) + list := idx.Lookup(testBlob.Handle(), nil) if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list) } result := list[0] rtest.Equals(t, testBlob, result) - list2 := idx2.Lookup(testBlob.BlobHandle, nil) + list2 := idx2.Lookup(testBlob.Handle(), nil) if len(list2) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list2), list2) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list2), list2) } result2 := list2[0] @@ -82,21 +83,21 @@ func TestIndexSerialize(t *testing.T) { } // add more blobs to idx - newtests := []restic.PackedBlob{} + newtests := []*pack.PackedBlob{} for i := 0; i < 10; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 10; j++ { length := uint(i*100 + j) - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, }, - PackID: packID, } blobs = append(blobs, pb.Blob) newtests = append(newtests, pb) @@ -127,9 +128,9 @@ func TestIndexSerialize(t *testing.T) { // all new blobs must be in the index for _, testBlob := range newtests { - list := idx3.Lookup(testBlob.BlobHandle, nil) + list := idx3.Lookup(testBlob.Handle(), nil) if len(list) != 1 { - t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list) + t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list) } blob := list[0] @@ -145,12 +146,12 @@ func TestIndexSize(t *testing.T) { blobCount := 100 for i := 0; i < packs; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < blobCount; j++ { length := uint(i*100 + j) - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, @@ -293,15 +294,15 @@ func TestIndexUnserialize(t *testing.T) { t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) - rtest.Equals(t, test.packID, blob.PackID) - rtest.Equals(t, test.tpe, blob.Type) - rtest.Equals(t, test.offset, blob.Offset) - rtest.Equals(t, test.length, blob.Length) + rtest.Equals(t, test.packID, blob.PackID()) + rtest.Equals(t, test.tpe, blob.Blob.Type) + rtest.Equals(t, test.offset, blob.Blob.Offset) + rtest.Equals(t, test.length, blob.Blob.Length) switch task.version { case 1: - rtest.Equals(t, uint(0), blob.UncompressedLength) + rtest.Equals(t, uint(0), blob.Blob.UncompressedLength) case 2: - rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength) + rtest.Equals(t, test.uncompressedLength, blob.Blob.UncompressedLength) default: t.Fatal("Invalid index version") } @@ -313,20 +314,20 @@ func TestIndexUnserialize(t *testing.T) { } for _, blob := range blobs { - b, ok := exampleLookupTest.blobs[blob.ID] + b, ok := exampleLookupTest.blobs[blob.Handle().ID] if !ok { - t.Errorf("unexpected blob %v found", blob.ID.Str()) + t.Errorf("unexpected blob %v found", blob.Handle().ID.String()) } - if blob.Type != b { - t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) + if blob.Blob.Type != b { + t.Errorf("unexpected type for blob %v: want %v, got %v", blob.Handle().ID.String(), b, blob.Blob.Type) } } } } -func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) { +func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []*pack.PackedBlob) { for pb := range idx.Values() { - if pb.PackID.Equal(id) { + if pb.PackID().Equal(id) { pbs = append(pbs, pb) } } @@ -401,7 +402,7 @@ func TestIndexPacks(t *testing.T) { for i := 0; i < 20; i++ { packID := restic.NewRandomID() - idx.StorePack(packID, restic.Blobs{ + idx.StorePack(packID, pack.Blobs{ { BlobHandle: restic.NewRandomBlobHandle(), Offset: 0, @@ -433,12 +434,12 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupB // create index with given number of pack files for i := 0; i < packfiles; i++ { packID := NewRandomTestID(rng) - var blobs restic.Blobs + var blobs pack.Blobs offset := 0 for offset < maxPackSize { size := 2000 + rng.Intn(4*1024*1024) id := NewRandomTestID(rng) - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: id, @@ -482,7 +483,7 @@ func BenchmarkIndexHasKnown(b *testing.B) { idx, _ := createRandomIndex(rand.New(rand.NewSource(0)), 200000) handles := make([]restic.BlobHandle, 0, 100000) for handle := range idx.Values() { - handles = append(handles, handle.BlobHandle) + handles = append(handles, handle.Handle()) if len(handles) == cap(handles) { break } @@ -517,14 +518,14 @@ func BenchmarkIndexAllocParallel(b *testing.B) { } func TestIndexHas(t *testing.T) { - tests := []restic.PackedBlob{} + tests := []*pack.PackedBlob{} idx := index.NewIndex() // create 50 packs with 20 blobs each for i := 0; i < 50; i++ { packID := restic.NewRandomID() - var blobs restic.Blobs + var blobs pack.Blobs pos := uint(0) for j := 0; j < 20; j++ { @@ -534,14 +535,14 @@ func TestIndexHas(t *testing.T) { // test a mix of compressed and uncompressed packs uncompressedLength = 2 * length } - pb := restic.PackedBlob{ - Blob: restic.Blob{ + pb := &pack.PackedBlob{ + Pack: packID, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Offset: pos, Length: length, UncompressedLength: uncompressedLength, }, - PackID: packID, } blobs = append(blobs, pb.Blob) tests = append(tests, pb) @@ -551,11 +552,11 @@ func TestIndexHas(t *testing.T) { } for _, testBlob := range tests { - rtest.Assert(t, idx.Has(testBlob.BlobHandle), "Index reports not having data blob added to it") + rtest.Assert(t, idx.Has(testBlob.Handle()), "Index reports not having data blob added to it") } rtest.Assert(t, !idx.Has(restic.NewRandomBlobHandle()), "Index reports having a data blob not added to it") - rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob") + rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].Handle().ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob") } func TestMixedEachByPack(t *testing.T) { @@ -566,7 +567,7 @@ func TestMixedEachByPack(t *testing.T) { for i := 0; i < 50; i++ { packID := restic.NewRandomID() expected[packID] = 1 - blobs := restic.Blobs{ + blobs := pack.Blobs{ { BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, Offset: 0, @@ -608,7 +609,7 @@ func TestEachByPackIgnoes(t *testing.T) { } else { expected[packID] = 1 } - blobs := restic.Blobs{ + blobs := pack.Blobs{ { BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()}, Offset: 0, diff --git a/internal/repository/index/master_index.go b/internal/repository/index/master_index.go index 84db0d962..f163f72e7 100644 --- a/internal/repository/index/master_index.go +++ b/internal/repository/index/master_index.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" @@ -39,10 +40,11 @@ func (mi *MasterIndex) clearPendingBlobs() { } // Lookup queries all known Indexes for the ID and returns all matches. -func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) { +func (mi *MasterIndex) Lookup(bh restic.BlobHandle) []*pack.PackedBlob { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() + var pbs []*pack.PackedBlob for _, idx := range mi.idx { pbs = idx.Lookup(bh, pbs) } @@ -145,12 +147,12 @@ func (mi *MasterIndex) Insert(idx *Index) { } // StorePack remembers the id and pack in the index. -func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs restic.Blobs, r restic.SaverUnpacked[restic.FileType]) error { +func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs pack.Blobs, r restic.SaverUnpacked[restic.FileType]) error { mi.storePack(id, blobs) return mi.saveFullIndex(ctx, r) } -func (mi *MasterIndex) storePack(id restic.ID, blobs restic.Blobs) { +func (mi *MasterIndex) storePack(id restic.ID, blobs pack.Blobs) { mi.idxMutex.Lock() defer mi.idxMutex.Unlock() @@ -218,8 +220,8 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index { // Values returns an iterator over all blobs known to the index. This blocks any // modification of the index. -func (mi *MasterIndex) Values() iter.Seq[restic.PackedBlob] { - return func(yield func(restic.PackedBlob) bool) { +func (mi *MasterIndex) Values() iter.Seq[*pack.PackedBlob] { + return func(yield func(*pack.PackedBlob) bool) { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() @@ -663,13 +665,13 @@ func (mi *MasterIndex) saveFullIndex(ctx context.Context, r restic.SaverUnpacked } // ListPacks returns the blobs of the specified pack files grouped by pack file. -func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { - out := make(chan restic.PackBlobs) +func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan PackBlobs { + out := make(chan PackBlobs) go func() { defer close(out) // only resort a part of the index to keep the memory overhead bounded for i := byte(0); i < 16; i++ { - packBlob := make(map[restic.ID]restic.Blobs) + packBlob := make(map[restic.ID]pack.Blobs) for pack := range packs { if pack[0]&0xf == i { packBlob[pack] = nil @@ -682,8 +684,9 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan if ctx.Err() != nil { return } - if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i { - packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob) + if packs.Has(pb.PackID()) && pb.PackID()[0]&0xf == i { + id := pb.PackID() + packBlob[id] = append(packBlob[id], pb.Blob) } } @@ -692,7 +695,7 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan // allow GC packBlob[packID] = nil select { - case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}: + case out <- PackBlobs{PackID: packID, Blobs: pbs}: case <-ctx.Done(): return } diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index 56d01a503..ed320cc3b 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/progress" @@ -24,18 +25,18 @@ func TestMasterIndex(t *testing.T) { bhInIdx2 := restic.NewRandomBlobHandle() bhInIdx12 := restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob} - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx1, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx2, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -43,9 +44,9 @@ func TestMasterIndex(t *testing.T) { }, } - blob12a := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob12a := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx12, Length: uint(crypto.CiphertextLength(123)), Offset: 110, @@ -53,9 +54,9 @@ func TestMasterIndex(t *testing.T) { }, } - blob12b := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob12b := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx12, Length: uint(crypto.CiphertextLength(123)), Offset: 50, @@ -64,12 +65,12 @@ func TestMasterIndex(t *testing.T) { } idx1 := index.NewIndex() - idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) - idx1.StorePack(blob12a.PackID, restic.Blobs{blob12a.Blob}) + idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) + idx1.StorePack(blob12a.PackID(), pack.Blobs{blob12a.Blob}) idx2 := index.NewIndex() - idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) - idx2.StorePack(blob12b.PackID, restic.Blobs{blob12b.Blob}) + idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) + idx2.StorePack(blob12b.PackID(), pack.Blobs{blob12b.Blob}) mIdx := index.NewMasterIndex() mIdx.Insert(idx1) @@ -77,7 +78,7 @@ func TestMasterIndex(t *testing.T) { // test idInIdx1 blobs := mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) size, found := mIdx.LookupSize(bhInIdx1) rtest.Equals(t, true, found) @@ -85,7 +86,7 @@ func TestMasterIndex(t *testing.T) { // test idInIdx2 blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) size, found = mIdx.LookupSize(bhInIdx2) rtest.Equals(t, true, found) @@ -95,19 +96,20 @@ func TestMasterIndex(t *testing.T) { blobs = mIdx.Lookup(bhInIdx12) rtest.Equals(t, 2, len(blobs)) - // test Lookup result for blob12a - found = false - if blobs[0] == blob12a || blobs[1] == blob12a { - found = true + containsPackedBlob := func(list []*pack.PackedBlob, want *pack.PackedBlob) bool { + for _, b := range list { + if b.PackID().Equal(want.PackID()) && b.Blob == want.Blob { + return true + } + } + return false } - rtest.Assert(t, found, "blob12a not found in result") + + // test Lookup result for blob12a + rtest.Assert(t, containsPackedBlob(blobs, blob12a), "blob12a not found in result") // test Lookup result for blob12b - found = false - if blobs[0] == blob12b || blobs[1] == blob12b { - found = true - } - rtest.Assert(t, found, "blob12a not found in result") + rtest.Assert(t, containsPackedBlob(blobs, blob12b), "blob12b not found in result") size, found = mIdx.LookupSize(bhInIdx12) rtest.Equals(t, true, found) @@ -135,7 +137,7 @@ func TestMasterIndexAddPending(t *testing.T) { // Test AddPending: try to add a blob that's already in an index (should return false) bhInIndex := restic.NewRandomBlobHandle() idx := index.NewIndex() - idx.StorePack(restic.NewRandomID(), restic.Blobs{{ + idx.StorePack(restic.NewRandomID(), pack.Blobs{{ BlobHandle: bhInIndex, Length: uint(crypto.CiphertextLength(50)), Offset: 0, @@ -173,14 +175,14 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) { // Store the blob in a pack packID := restic.NewRandomID() - blob := restic.Blob{ + blob := pack.Blob{ BlobHandle: bhPending, Length: uint(crypto.CiphertextLength(75)), Offset: 0, UncompressedLength: 75, } saver := &noopSaver{} - err := mIdx.StorePack(context.Background(), packID, restic.Blobs{blob}, saver) + err := mIdx.StorePack(context.Background(), packID, pack.Blobs{blob}, saver) rtest.OK(t, err) // Verify it is still found @@ -191,8 +193,8 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) { // Verify the blob can be found via Lookup from the index blobs := mIdx.Lookup(bhPending) rtest.Assert(t, len(blobs) > 0, "blob should be found in index after StorePack") - rtest.Equals(t, packID, blobs[0].PackID) - rtest.Equals(t, bhPending, blobs[0].BlobHandle) + rtest.Equals(t, packID, blobs[0].PackID()) + rtest.Equals(t, bhPending, blobs[0].Handle()) // Test that adding the same blob as pending again fails (it's now in index) added = mIdx.AddPending(bhPending, 100) @@ -203,18 +205,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) { bhInIdx1 := restic.NewRandomBlobHandle() bhInIdx2 := restic.NewRandomBlobHandle() - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx1, Length: 10, Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhInIdx2, Length: 100, Offset: 10, @@ -223,10 +225,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) { } idx1 := index.NewIndex() - idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) + idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) idx2 := index.NewIndex() - idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) + idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) mIdx := index.NewMasterIndex() mIdx.Insert(idx1) @@ -246,18 +248,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) { rtest.Equals(t, 2, blobCount) blobs := mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) blobs = mIdx.Lookup(restic.NewRandomBlobHandle()) rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id") // merge another index containing identical blobs idx3 := index.NewIndex() - idx3.StorePack(blob1.PackID, restic.Blobs{blob1.Blob}) - idx3.StorePack(blob2.PackID, restic.Blobs{blob2.Blob}) + idx3.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob}) + idx3.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob}) mIdx.Insert(idx3) finalIndexes, idxCount, newIDs := index.TestMergeIndex(t, mIdx) @@ -268,10 +270,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) { // Index should have same entries as before! blobs = mIdx.Lookup(bhInIdx1) - rtest.Equals(t, []restic.PackedBlob{blob1}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs) blobs = mIdx.Lookup(bhInIdx2) - rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) + rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs) blobCount = 0 for range mIdx.Values() { @@ -448,9 +450,9 @@ func testIndexSave(t *testing.T, version uint) { idx := index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) - blobs := make(map[restic.PackedBlob]struct{}) + blobs := make(map[pack.PackedBlob]struct{}) for pb := range idx.Values() { - blobs[pb] = struct{}{} + blobs[*pb] = struct{}{} } rtest.OK(t, test.saver(idx, unpacked)) @@ -458,8 +460,8 @@ func testIndexSave(t *testing.T, version uint) { rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) for pb := range idx.Values() { - if _, ok := blobs[pb]; ok { - delete(blobs, pb) + if _, ok := blobs[*pb]; ok { + delete(blobs, *pb) } else { t.Fatalf("unexpected blobs %v", pb) } @@ -481,9 +483,9 @@ func testIndexSavePartial(t *testing.T, version uint) { // capture blob list before adding fourth snapshot idx := index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) - blobs := make(map[restic.PackedBlob]struct{}) + blobs := make(map[pack.PackedBlob]struct{}) for pb := range idx.Values() { - blobs[pb] = struct{}{} + blobs[*pb] = struct{}{} } // add+remove new snapshot and track its pack files @@ -502,8 +504,8 @@ func testIndexSavePartial(t *testing.T, version uint) { idx = index.NewMasterIndex() rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) for pb := range idx.Values() { - if _, ok := blobs[pb]; ok { - delete(blobs, pb) + if _, ok := blobs[*pb]; ok { + delete(blobs, *pb) } else { t.Fatalf("unexpected blobs %v", pb) } @@ -516,7 +518,7 @@ func testIndexSavePartial(t *testing.T, version uint) { checker.TestCheckRepo(t, repo) } -func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[restic.PackedBlob]struct{} { +func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[pack.PackedBlob]struct{} { p := progress.NewCounter(0, 0, nil) rtest.OK(t, master.Load(context.TODO(), repo, p, nil)) v, max := p.Get() @@ -525,10 +527,10 @@ func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, ma return collectBlobs(master) } -func collectBlobs(master *index.MasterIndex) map[restic.PackedBlob]struct{} { - s := make(map[restic.PackedBlob]struct{}) +func collectBlobs(master *index.MasterIndex) map[pack.PackedBlob]struct{} { + s := make(map[pack.PackedBlob]struct{}) for pb := range master.Values() { - s[pb] = struct{}{} + s[*pb] = struct{}{} } return s } @@ -588,17 +590,17 @@ func TestRewriteOversizedIndex(t *testing.T) { return idx.Len(restic.DataBlob) > 2*fullIndexCount } - var blobs restic.Blobs + var blobs pack.Blobs // build oversized index idx := index.NewIndex() numPacks := 5 for p := 0; p < numPacks; p++ { packID := restic.NewRandomID() - packBlobs := make(restic.Blobs, 0, fullIndexCount) + packBlobs := make(pack.Blobs, 0, fullIndexCount) for i := 0; i < fullIndexCount; i++ { - blob := restic.Blob{ + blob := pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: restic.NewRandomID(), @@ -645,17 +647,17 @@ func TestRewriteSplitPacks(t *testing.T) { bh2 := restic.NewRandomBlobHandle() bhOther := restic.NewRandomBlobHandle() - blob1 := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blob1 := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bh1, Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blob2 := restic.PackedBlob{ - PackID: blob1.PackID, - Blob: restic.Blob{ + blob2 := &pack.PackedBlob{ + Pack: blob1.PackID(), + Blob: pack.Blob{ BlobHandle: bh2, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -663,9 +665,9 @@ func TestRewriteSplitPacks(t *testing.T) { }, } // used to force index repacking - blobOther := restic.PackedBlob{ - PackID: restic.NewRandomID(), - Blob: restic.Blob{ + blobOther := &pack.PackedBlob{ + Pack: restic.NewRandomID(), + Blob: pack.Blob{ BlobHandle: bhOther, Length: uint(crypto.CiphertextLength(100)), Offset: 10, @@ -673,25 +675,25 @@ func TestRewriteSplitPacks(t *testing.T) { } mi := index.NewMasterIndex() - rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID), nil, nil, index.MasterIndexRewriteOpts{})) + rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID()), nil, nil, index.MasterIndexRewriteOpts{})) mi = index.NewMasterIndex() rtest.OK(t, mi.Load(context.TODO(), repo, nil, nil)) // test that all blobs are still in the index - for _, blob := range []restic.PackedBlob{blob1, blob2} { - blobs := mi.Lookup(blob.BlobHandle) - rtest.Equals(t, []restic.PackedBlob{blob}, blobs) + for _, blob := range []*pack.PackedBlob{blob1, blob2} { + blobs := mi.Lookup(blob.Handle()) + rtest.Equals(t, []*pack.PackedBlob{blob}, blobs) } - blobs := mi.Lookup(blobOther.BlobHandle) + blobs := mi.Lookup(blobOther.Handle()) rtest.Equals(t, nil, blobs) } @@ -713,17 +715,17 @@ func TestRewriteFullPacks(t *testing.T) { packA := restic.NewRandomID() packB := restic.NewRandomID() - blobA := restic.PackedBlob{ - PackID: packA, - Blob: restic.Blob{ + blobA := &pack.PackedBlob{ + Pack: packA, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Length: uint(crypto.CiphertextLength(10)), Offset: 0, }, } - blobB := restic.PackedBlob{ - PackID: packB, - Blob: restic.Blob{ + blobB := &pack.PackedBlob{ + Pack: packB, + Blob: pack.Blob{ BlobHandle: restic.NewRandomBlobHandle(), Length: uint(crypto.CiphertextLength(50)), Offset: 0, @@ -731,11 +733,11 @@ func TestRewriteFullPacks(t *testing.T) { } mi := index.NewMasterIndex() - rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packA, pack.Blobs{blobA.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) - rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked)) + rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked)) rtest.OK(t, mi.Flush(context.TODO(), unpacked)) indexIDs := mi.IDs() @@ -750,6 +752,6 @@ func TestRewriteFullPacks(t *testing.T) { rtest.Equals(t, 2, len(afterRewrite)) rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs))) - rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle)) - rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle)) + rtest.Equals(t, []*pack.PackedBlob{blobA}, mi2.Lookup(blobA.Handle())) + rtest.Equals(t, []*pack.PackedBlob{blobB}, mi2.Lookup(blobB.Handle())) } diff --git a/internal/repository/index_list.go b/internal/repository/index_list.go index 18b40f114..c6d003539 100644 --- a/internal/repository/index_list.go +++ b/internal/repository/index_list.go @@ -28,7 +28,7 @@ func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.Load if ctx.Err() != nil { return ctx.Err() } - if !yield(IndexBlob{Handle: blob.BlobHandle}) { + if !yield(IndexBlob{Handle: blob.Handle()}) { return stopIteration } } diff --git a/internal/repository/index_testutil_test.go b/internal/repository/index_testutil_test.go index cc44774c4..9e3a158ae 100644 --- a/internal/repository/index_testutil_test.go +++ b/internal/repository/index_testutil_test.go @@ -1,14 +1,15 @@ package repository import ( + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" ) // BlobsInPack returns index entries for blobs stored in packID, sorted by offset. -func BlobsInPack(repo *Repository, packID restic.ID) restic.Blobs { - var blobs restic.Blobs +func BlobsInPack(repo *Repository, packID restic.ID) pack.Blobs { + var blobs pack.Blobs for pb := range repo.idx.Values() { - if pb.PackID.Equal(packID) { + if pb.PackID().Equal(packID) { blobs = append(blobs, pb.Blob) } } diff --git a/internal/repository/pack/blob.go b/internal/repository/pack/blob.go new file mode 100644 index 000000000..5ec8e6680 --- /dev/null +++ b/internal/repository/pack/blob.go @@ -0,0 +1,32 @@ +package pack + +import ( + "fmt" + + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/restic" +) + +// Blob is one part of a file or a tree with pack layout information. +type Blob struct { + restic.BlobHandle + Length uint + Offset uint + UncompressedLength uint +} + +func (b Blob) String() string { + return fmt.Sprintf("", + b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength) +} + +func (b Blob) DataLength() uint { + if b.UncompressedLength != 0 { + return b.UncompressedLength + } + return uint(crypto.PlaintextLength(int(b.Length))) +} + +func (b Blob) IsCompressed() bool { + return b.UncompressedLength != 0 +} diff --git a/internal/repository/pack/blobs.go b/internal/repository/pack/blobs.go new file mode 100644 index 000000000..04fe04d84 --- /dev/null +++ b/internal/repository/pack/blobs.go @@ -0,0 +1,15 @@ +package pack + +import ( + "cmp" + "slices" +) + +// Blobs is a list of blobs with pack layout information (offset, length, ...). +type Blobs []Blob + +func (b Blobs) Sort() { + slices.SortFunc(b, func(a, b Blob) int { + return cmp.Compare(a.Offset, b.Offset) + }) +} diff --git a/internal/repository/pack/blobs_test.go b/internal/repository/pack/blobs_test.go new file mode 100644 index 000000000..b345e482f --- /dev/null +++ b/internal/repository/pack/blobs_test.go @@ -0,0 +1,24 @@ +package pack + +import ( + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestBlobsSort(t *testing.T) { + blobs := Blobs{ + {Offset: 100}, + {Offset: 0}, + {Offset: 50}, + } + blobs.Sort() + rtest.Equals(t, uint(0), blobs[0].Offset) + rtest.Equals(t, uint(50), blobs[1].Offset) + rtest.Equals(t, uint(100), blobs[2].Offset) +} + +func TestBlobsSortNilSlice(t *testing.T) { + var blobs Blobs + blobs.Sort() +} diff --git a/internal/repository/pack/pack.go b/internal/repository/pack/pack.go index 25645bddd..b23043e7f 100644 --- a/internal/repository/pack/pack.go +++ b/internal/repository/pack/pack.go @@ -21,7 +21,7 @@ var ErrBroken = errors.New("packer cannot be used after a write error") // Packer is used to create a new Pack. type Packer struct { - blobs restic.Blobs + blobs []Blob bytes uint k *crypto.Key @@ -56,7 +56,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL return n, p.err } - c := restic.Blob{ + c := Blob{ BlobHandle: restic.BlobHandle{Type: t, ID: id}, Length: uint(n), Offset: p.bytes, @@ -129,7 +129,7 @@ func (p *Packer) Finalize() error { return nil } -func verifyHeader(k *crypto.Key, header []byte, expected restic.Blobs) error { +func verifyHeader(k *crypto.Key, header []byte, expected []Blob) error { // do not offer a way to skip the pack header verification, as pack headers are usually small enough // to not result in a significant performance impact @@ -157,7 +157,7 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func makeHeader(blobs restic.Blobs) ([]byte, error) { +func makeHeader(blobs []Blob) ([]byte, error) { buf := make([]byte, 0, len(blobs)*int(entrySize)) for _, b := range blobs { @@ -232,7 +232,7 @@ func (p *Packer) HeaderFull() bool { } // Blobs returns the slice of blobs that have been written. -func (p *Packer) Blobs() restic.Blobs { +func (p *Packer) Blobs() Blobs { p.m.Lock() defer p.m.Unlock() @@ -348,7 +348,7 @@ func (e InvalidFileError) Error() string { // List returns the list of entries found in a pack file and the length of the // header (including header size and crypto overhead) -func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrSize uint32, err error) { +func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries Blobs, hdrSize uint32, err error) { buf, err := readHeader(rd, size) if err != nil { return nil, 0, err @@ -367,7 +367,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS } // might over allocate a bit if all blobs have EntrySize but only by a few percent - entries = make(restic.Blobs, 0, uint(len(buf))/plainEntrySize) + entries = make(Blobs, 0, uint(len(buf))/plainEntrySize) pos := uint(0) for len(buf) > 0 { @@ -385,7 +385,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS return entries, hdrSize, nil } -func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) { +func parseHeaderEntry(p []byte) (b Blob, size uint, err error) { l := uint(len(p)) size = plainEntrySize if l < plainEntrySize { @@ -427,7 +427,7 @@ func CalculateEntrySize(compressed bool) int { return int(plainEntrySize) } -func CalculateHeaderSize(blobs restic.Blobs) int { +func CalculateHeaderSize(blobs Blobs) int { size := headerSize for _, blob := range blobs { size += CalculateEntrySize(blob.IsCompressed()) @@ -439,10 +439,10 @@ func CalculateHeaderSize(blobs restic.Blobs) int { // If onlyHdr is set to true, only the size of the header is returned // Note that this function only gives correct sizes, if there are no // duplicates in the index. -func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { +func Size(ctx context.Context, idx restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { packSize := make(map[restic.ID]int64) - err := mi.ListBlobs(ctx, func(blob restic.PackBlob) { + err := idx.ListBlobs(ctx, func(blob restic.PackBlob) { packID := blob.PackID() size, ok := packSize[packID] if !ok { diff --git a/internal/repository/pack/pack_internal_test.go b/internal/repository/pack/pack_internal_test.go index 186450e1c..d9a15298a 100644 --- a/internal/repository/pack/pack_internal_test.go +++ b/internal/repository/pack/pack_internal_test.go @@ -182,7 +182,7 @@ func TestReadRecords(t *testing.T) { func TestUnpackedVerification(t *testing.T) { // create random keys k := crypto.NewRandomKey() - blobs := restic.Blobs{ + blobs := []Blob{ { BlobHandle: restic.NewRandomBlobHandle(), Length: 42, diff --git a/internal/repository/pack/packedblob.go b/internal/repository/pack/packedblob.go new file mode 100644 index 000000000..e5d3e3588 --- /dev/null +++ b/internal/repository/pack/packedblob.go @@ -0,0 +1,19 @@ +package pack + +import "github.com/restic/restic/internal/restic" + +// PackedBlob is one index entry for a blob in a pack (may be duplicate across indexes). +type PackedBlob struct { + Pack restic.ID + Blob Blob +} + +func (pb *PackedBlob) PackID() restic.ID { return pb.Pack } + +func (pb *PackedBlob) Handle() restic.BlobHandle { return pb.Blob.BlobHandle } + +func (pb *PackedBlob) CiphertextLength() uint { return pb.Blob.Length } + +func (pb *PackedBlob) PlaintextLength() uint { return pb.Blob.DataLength() } + +func (pb *PackedBlob) IsCompressed() bool { return pb.Blob.IsCompressed() } diff --git a/internal/repository/repack.go b/internal/repository/repack.go index a4472c91e..d8e351089 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -7,6 +7,8 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" @@ -80,11 +82,11 @@ func repack( } var keepMutex sync.Mutex - downloadQueue := make(chan restic.PackBlobs) + downloadQueue := make(chan index.PackBlobs) wg.Go(func() error { defer close(downloadQueue) for pbs := range repo.listPacksFromIndex(wgCtx, packs) { - var packBlobs restic.Blobs + var packBlobs pack.Blobs keepMutex.Lock() // filter out unnecessary blobs for _, entry := range pbs.Blobs { @@ -96,7 +98,7 @@ func repack( keepMutex.Unlock() select { - case downloadQueue <- restic.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}: + case downloadQueue <- index.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}: case <-wgCtx.Done(): return wgCtx.Err() } diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index a7d04bbc2..1c27acf5b 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -6,6 +6,7 @@ import ( "io" "slices" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" ) @@ -71,8 +72,8 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe return nil } -func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]restic.Blobs, error) { - packToBlobs := make(map[restic.ID]restic.Blobs) +func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]pack.Blobs, error) { + packToBlobs := make(map[restic.ID]pack.Blobs) err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { if ids.Has(id) { @@ -90,7 +91,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe return packToBlobs, nil } -func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs restic.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error { +func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs pack.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error { err := repo.loadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { printer.E("failed to load blob %v: %v", blob.ID, err) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index d28ce64c9..e7d939894 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -213,7 +213,7 @@ type haver interface { } // sortCachedPacksFirst moves all cached pack files to the front of blobs. -func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { +func sortCachedPacksFirst(cache haver, blobs []*pack.PackedBlob) { if cache == nil { return } @@ -224,10 +224,10 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { } cached := blobs[:0] - noncached := make([]restic.PackedBlob, 0, len(blobs)/2) + noncached := make([]*pack.PackedBlob, 0, len(blobs)/2) for _, blob := range blobs { - if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { + if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID().String()}) { cached = append(cached, blob) continue } @@ -256,7 +256,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. if err != nil { if r.cache != nil { for _, blob := range blobs { - h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()} // ignore errors as there's not much we can do here _ = r.cache.Forget(h) } @@ -267,28 +267,28 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. return buf, err } -func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, buf []byte) ([]byte, error) { +func (r *Repository) loadBlob(ctx context.Context, blobs []*pack.PackedBlob, buf []byte) ([]byte, error) { var lastError error for _, blob := range blobs { - debug.Log("blob %v found: %v", blob.BlobHandle, blob) + debug.Log("blob %v found: %v", blob.Handle(), blob) // load blob from pack - h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()} switch { - case cap(buf) < int(blob.Length): - buf = make([]byte, blob.Length) - case len(buf) != int(blob.Length): - buf = buf[:blob.Length] + case cap(buf) < int(blob.Blob.Length): + buf = make([]byte, blob.Blob.Length) + case len(buf) != int(blob.Blob.Length): + buf = buf[:blob.Blob.Length] } - _, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) + _, err := backend.ReadAt(ctx, r.be, h, int64(blob.Blob.Offset), buf) if err != nil { debug.Log("error loading blob %v: %v", blob, err) lastError = err continue } - it := newPackBlobIterator(blob.PackID, newByteReader(buf), blob.Offset, restic.Blobs{blob.Blob}, r.key, r.getZstdDecoder()) + it := newPackBlobIterator(blob.PackID(), newByteReader(buf), blob.Blob.Offset, pack.Blobs{blob.Blob}, r.key, r.getZstdDecoder()) pbv, err := it.Next() if err == nil { @@ -314,7 +314,7 @@ func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, bu return nil, lastError } - return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs)) + return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].Handle(), len(blobs)) } func (r *Repository) getZstdEncoder() *zstd.Encoder { @@ -673,8 +673,8 @@ func (r *Repository) Connections() uint { func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob { entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) out := make([]restic.PackBlob, len(entries)) - for i, pb := range entries { - out[i] = restic.AsPackBlob(pb) + for i, e := range entries { + out[i] = e } return out } @@ -691,13 +691,13 @@ func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) er if ctx.Err() != nil { return ctx.Err() } - fn(restic.AsPackBlob(blob)) + fn(blob) } return nil } // listPacksFromIndex returns index entries for the given packs, grouped by pack file. -func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { +func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan index.PackBlobs { return r.idx.ListPacks(ctx, packs) } @@ -964,7 +964,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic } // listPack returns blob entries from the pack file header including offsets. -func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) { +func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (pack.Blobs, error) { h := backend.Handle{Type: restic.PackFile, Name: id.String()} entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) @@ -977,7 +977,7 @@ func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (re // retry on error entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) } - return entries, err + return pack.Blobs(entries), err } // ListPackHandles returns the blob handles stored in the pack file header. @@ -1074,12 +1074,12 @@ func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, ha return r.loadBlobsFromPack(ctx, packID, blobs, handleBlobFn) } -func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (restic.Blobs, error) { - blobs := make(restic.Blobs, 0, len(handles)) +func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (pack.Blobs, error) { + blobs := make(pack.Blobs, 0, len(handles)) for _, h := range handles { found := false for _, pb := range r.idx.Lookup(h) { - if pb.PackID.Equal(packID) { + if pb.PackID().Equal(packID) { blobs = append(blobs, pb.Blob) found = true break @@ -1092,11 +1092,11 @@ func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) return blobs, nil } -func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return streamPack(ctx, r.be.Load, r.LoadBlob, r.getZstdDecoder(), r.key, packID, blobs, handleBlobFn) } -func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do return nil @@ -1139,7 +1139,7 @@ func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn return streamPackPart(ctx, beLoad, loadBlobFn, dec, key, packID, blobs[lowerIdx:], handleBlobFn) } -func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: blobs[0].Type.IsMetadata()} dataStart := blobs[0].Offset @@ -1249,7 +1249,7 @@ type packBlobIterator struct { rd discardReader currentOffset uint - blobs restic.Blobs + blobs pack.Blobs key *crypto.Key dec *zstd.Decoder @@ -1265,7 +1265,7 @@ type packBlobValue struct { var errPackEOF = errors.New("reached EOF of pack file") func newPackBlobIterator(packID restic.ID, rd discardReader, currentOffset uint, - blobs restic.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator { + blobs pack.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator { return &packBlobIterator{ packID: packID, rd: rd, diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index 31aed6f64..226c3c237 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -17,6 +17,7 @@ import ( "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,7 +28,7 @@ func (c mapcache) Has(h backend.Handle) bool { return c[h] } func TestSortCachedPacksFirst(t *testing.T) { var ( - blobs, sorted [100]restic.PackedBlob + blobs, sorted [100]*pack.PackedBlob cache = make(mapcache) r = rand.New(rand.NewSource(1261)) @@ -36,7 +37,7 @@ func TestSortCachedPacksFirst(t *testing.T) { for i := 0; i < len(blobs); i++ { var id restic.ID r.Read(id[:]) - blobs[i] = restic.PackedBlob{PackID: id} + blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}} if i%3 == 0 { h := backend.Handle{Name: id.String(), Type: backend.PackFile} @@ -46,8 +47,8 @@ func TestSortCachedPacksFirst(t *testing.T) { copy(sorted[:], blobs[:]) sort.SliceStable(sorted[:], func(i, j int) bool { - hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()} - hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()} + hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID().String()} + hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID().String()} return cache.Has(hi) && !cache.Has(hj) }) @@ -59,7 +60,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { const nblobs = 512 // Corresponds to a file of ca. 2GB. var ( - blobs [nblobs]restic.PackedBlob + blobs [nblobs]*pack.PackedBlob cache = make(mapcache) r = rand.New(rand.NewSource(1261)) ) @@ -67,7 +68,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { for i := 0; i < nblobs; i++ { var id restic.ID r.Read(id[:]) - blobs[i] = restic.PackedBlob{PackID: id} + blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}} if i%3 == 0 { h := backend.Handle{Name: id.String(), Type: backend.PackFile} @@ -75,7 +76,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { } } - var cpy [nblobs]restic.PackedBlob + var cpy [nblobs]*pack.PackedBlob b.ReportAllocs() b.ResetTimer() @@ -96,7 +97,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) { idx := index.NewIndex() for i := 0; i < 5000; i++ { - idx.StorePack(restic.NewRandomID(), restic.Blobs{ + idx.StorePack(restic.NewRandomID(), pack.Blobs{ { BlobHandle: restic.NewRandomBlobHandle(), Length: 1234, @@ -133,7 +134,7 @@ func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (* } // buildPackfileWithoutHeader returns a manually built pack file without a header. -func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs restic.Blobs, packfile []byte) { +func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs pack.Blobs, packfile []byte) { opts := []zstd.EOption{ // Set the compression level configured. zstd.WithEncoderLevel(zstd.SpeedDefault), @@ -173,7 +174,7 @@ func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) ciphertextLength := after - before - blobs = append(blobs, restic.Blob{ + blobs = append(blobs, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: id, @@ -280,19 +281,19 @@ func testStreamPack(t *testing.T, version uint) { // first, test regular usage t.Run("regular", func(t *testing.T) { tests := []struct { - blobs restic.Blobs + blobs pack.Blobs calls int shortFirstLoad bool }{ {packfileBlobs[1:2], 1, false}, {packfileBlobs[2:5], 1, false}, {packfileBlobs[2:8], 1, false}, - {restic.Blobs{ + {pack.Blobs{ packfileBlobs[0], packfileBlobs[4], packfileBlobs[2], }, 1, false}, - {restic.Blobs{ + {pack.Blobs{ packfileBlobs[0], packfileBlobs[len(packfileBlobs)-1], }, 2, false}, @@ -341,12 +342,12 @@ func testStreamPack(t *testing.T, version uint) { // next, test invalid uses, which should return an error t.Run("invalid", func(t *testing.T) { tests := []struct { - blobs restic.Blobs + blobs pack.Blobs err string }{ { // pass one blob several times - blobs: restic.Blobs{ + blobs: pack.Blobs{ packfileBlobs[3], packfileBlobs[8], packfileBlobs[3], @@ -357,7 +358,7 @@ func testStreamPack(t *testing.T, version uint) { { // pass something that's not a valid blob in the current pack file - blobs: restic.Blobs{ + blobs: pack.Blobs{ { Offset: 123, Length: 20000, @@ -368,7 +369,7 @@ func testStreamPack(t *testing.T, version uint) { { // pass a blob that's too small - blobs: restic.Blobs{ + blobs: pack.Blobs{ { Offset: 123, Length: 10, @@ -523,7 +524,7 @@ func TestStreamPackFallback(t *testing.T) { plaintext := rtest.Random(800, 42) blobID := restic.Hash(plaintext) - blobs := restic.Blobs{ + blobs := pack.Blobs{ { Length: uint(crypto.CiphertextLength(len(plaintext))), Offset: 0, diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index a775f9f43..6e8bbdbf0 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -400,7 +400,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { rtest.OK(t, err) for pb := range idx.Values() { - packID := pb.PackID + packID := pb.PackID() if _, ok := packEntries[packID]; !ok { packEntries[packID] = make(map[restic.ID]struct{}) } diff --git a/internal/restic/blob.go b/internal/restic/blob.go index 8d1e8a720..5fad7b3f6 100644 --- a/internal/restic/blob.go +++ b/internal/restic/blob.go @@ -1,46 +1,11 @@ package restic import ( - "cmp" "fmt" - "slices" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" ) -// Blob is one part of a file or a tree. -type Blob struct { - BlobHandle - Length uint - Offset uint - UncompressedLength uint -} - -func (b Blob) String() string { - return fmt.Sprintf("", - b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength) -} - -func (b Blob) DataLength() uint { - if b.UncompressedLength != 0 { - return b.UncompressedLength - } - return uint(crypto.PlaintextLength(int(b.Length))) -} - -func (b Blob) IsCompressed() bool { - return b.UncompressedLength != 0 -} - -type Blobs []Blob - -func (b Blobs) Sort() { - slices.SortFunc(b, func(a, b Blob) int { - return cmp.Compare(a.Offset, b.Offset) - }) -} - // PackBlob is one index entry for a blob in a pack file. // The interface intentionally omits the offset at which a blob is stored in the pack. // This ensures that pack file internals are not leaked. @@ -54,29 +19,6 @@ type PackBlob interface { IsCompressed() bool } -// PackedBlob is a blob stored within a file. -type PackedBlob struct { - Blob - PackID ID -} - -type packBlob struct { - PackedBlob -} - -func (pb packBlob) PackID() ID { return pb.PackedBlob.PackID } - -func (pb packBlob) Handle() BlobHandle { return pb.BlobHandle } - -func (pb packBlob) CiphertextLength() uint { return pb.Length } - -func (pb packBlob) PlaintextLength() uint { return pb.DataLength() } - -func (pb packBlob) IsCompressed() bool { return pb.Blob.IsCompressed() } - -// AsPackBlob returns a PackBlob view of a PackedBlob. -func AsPackBlob(pb PackedBlob) PackBlob { return packBlob{pb} } - // BlobHandle identifies a blob of a given type. type BlobHandle struct { ID ID diff --git a/internal/restic/blob_test.go b/internal/restic/blob_test.go index 36e045792..951872250 100644 --- a/internal/restic/blob_test.go +++ b/internal/restic/blob_test.go @@ -3,8 +3,6 @@ package restic import ( "encoding/json" "testing" - - rtest "github.com/restic/restic/internal/test" ) var blobTypeJSON = []struct { @@ -41,20 +39,3 @@ func TestBlobTypeJSON(t *testing.T) { } } } - -func TestBlobsSort(t *testing.T) { - blobs := Blobs{ - {Offset: 100}, - {Offset: 0}, - {Offset: 50}, - } - blobs.Sort() - rtest.Equals(t, uint(0), blobs[0].Offset) - rtest.Equals(t, uint(50), blobs[1].Offset) - rtest.Equals(t, uint(100), blobs[2].Offset) -} - -func TestBlobsSortNilSlice(t *testing.T) { - var blobs Blobs - blobs.Sort() -} diff --git a/internal/restic/repository.go b/internal/restic/repository.go index db969a41a..b3e688271 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -124,11 +124,6 @@ type SaverRemoverUnpacked[FT FileTypes] interface { RemoverUnpacked[FT] } -type PackBlobs struct { - PackID ID - Blobs Blobs -} - type TerminalCounterFactory interface { // NewCounterTerminalOnly returns a new progress counter that is only shown if stdout points to a // terminal. It is not shown if --quiet or --json is specified.