mirror of
https://github.com/restic/restic.git
synced 2026-06-06 08:59:44 +00:00
repository: move Blob, Blobs and PackedBlob to pack package
This removes them from the public interface. The latter now only provides the PackBlob interface, without being bound to the type used internally by the pack package.
This commit is contained in:
@@ -129,10 +129,11 @@ func (c *Checker) LoadIndex(ctx context.Context, p restic.TerminalCounterFactory
|
||||
}
|
||||
cnt++
|
||||
|
||||
if _, ok := packToIndex[blob.PackID]; !ok {
|
||||
packToIndex[blob.PackID] = restic.NewIDSet()
|
||||
packID := blob.PackID()
|
||||
if _, ok := packToIndex[packID]; !ok {
|
||||
packToIndex[packID] = restic.NewIDSet()
|
||||
}
|
||||
packToIndex[blob.PackID].Insert(id)
|
||||
packToIndex[packID].Insert(id)
|
||||
}
|
||||
|
||||
for pbs := range idx.EachByPack(ctx, restic.NewIDSet()) {
|
||||
@@ -259,7 +260,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
|
||||
type checkTask struct {
|
||||
id restic.ID
|
||||
size int64
|
||||
blobs restic.Blobs
|
||||
blobs pack.Blobs
|
||||
}
|
||||
ch := make(chan checkTask)
|
||||
|
||||
@@ -329,7 +330,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
|
||||
}
|
||||
|
||||
// checkPack reads a pack and checks the integrity of all blobs.
|
||||
func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
|
||||
if err != nil {
|
||||
if r.cache != nil {
|
||||
@@ -348,7 +349,7 @@ func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Bl
|
||||
return err
|
||||
}
|
||||
|
||||
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs pack.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
|
||||
type partialReadError struct {
|
||||
error
|
||||
@@ -465,7 +466,7 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs rest
|
||||
// Check if blob is contained in index and position is correct
|
||||
idxHas := false
|
||||
for _, pb := range r.idx.Lookup(blob.BlobHandle) {
|
||||
if pb.PackID.Equal(id) && pb.Blob == blob {
|
||||
if pb.PackID().Equal(id) && pb.Blob == blob {
|
||||
idxHas = true
|
||||
break
|
||||
}
|
||||
|
||||
@@ -123,7 +123,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami
|
||||
|
||||
checkPackSize(b.Blobs, len(buf), printer)
|
||||
|
||||
err = loadBlobs(ctx, opts, repo, id, b.Blobs, printer)
|
||||
err := loadBlobs(ctx, opts, repo, id, b.Blobs, printer)
|
||||
if err != nil {
|
||||
printer.E("error: %v", err)
|
||||
} else {
|
||||
@@ -146,7 +146,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
|
||||
func checkPackSize(blobs pack.Blobs, fileSize int, printer progress.Printer) {
|
||||
// track current size and offset
|
||||
var size, offset uint64
|
||||
|
||||
@@ -284,7 +284,7 @@ func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
|
||||
return out
|
||||
}
|
||||
|
||||
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
|
||||
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list pack.Blobs, printer progress.Printer) error {
|
||||
dec, err := zstd.NewReader(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
@@ -159,14 +159,15 @@ func (a *AssociatedSet[T]) All() iter.Seq2[restic.BlobHandle, T] {
|
||||
}
|
||||
|
||||
for pb := range a.idx.Values() {
|
||||
if _, ok := a.overflow[pb.BlobHandle]; ok {
|
||||
bh := pb.Handle()
|
||||
if _, ok := a.overflow[bh]; ok {
|
||||
// already reported via overflow set
|
||||
continue
|
||||
}
|
||||
|
||||
val, known := a.Get(pb.BlobHandle)
|
||||
val, known := a.Get(bh)
|
||||
if known {
|
||||
if !yield(pb.BlobHandle, val) {
|
||||
if !yield(bh, val) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
@@ -19,17 +20,17 @@ func (n *noopSaver) SaveUnpacked(_ context.Context, _ restic.FileType, buf []byt
|
||||
return restic.Hash(buf), nil
|
||||
}
|
||||
|
||||
func makeFakePackedBlob() (restic.BlobHandle, restic.PackedBlob) {
|
||||
func makeFakePackedBlob() (restic.BlobHandle, *pack.PackedBlob) {
|
||||
bh := restic.NewRandomBlobHandle()
|
||||
blob := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
pb := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bh,
|
||||
Length: uint(crypto.CiphertextLength(10)),
|
||||
Offset: 0,
|
||||
},
|
||||
}
|
||||
return bh, blob
|
||||
return bh, pb
|
||||
}
|
||||
|
||||
func list(bs *AssociatedSet[uint8]) restic.BlobHandles {
|
||||
@@ -40,7 +41,7 @@ func TestAssociatedSet(t *testing.T) {
|
||||
bh, blob := makeFakePackedBlob()
|
||||
|
||||
mi := NewMasterIndex()
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{}))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{}))
|
||||
test.OK(t, mi.Flush(context.TODO(), &noopSaver{}))
|
||||
|
||||
bs := NewAssociatedSet[uint8](mi)
|
||||
@@ -123,14 +124,14 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) {
|
||||
_, blob := makeFakePackedBlob()
|
||||
|
||||
mi := NewMasterIndex()
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob.PackID, restic.Blobs{blob.Blob}, &noopSaver{}))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob.PackID(), pack.Blobs{blob.Blob}, &noopSaver{}))
|
||||
test.OK(t, mi.Flush(context.TODO(), &noopSaver{}))
|
||||
|
||||
bs := NewAssociatedSet[uint8](mi)
|
||||
|
||||
// add new blobs to index after building the set
|
||||
of, blob2 := makeFakePackedBlob()
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, &noopSaver{}))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, &noopSaver{}))
|
||||
test.OK(t, mi.Flush(context.TODO(), &noopSaver{}))
|
||||
|
||||
// non-existent
|
||||
@@ -167,10 +168,10 @@ func TestAssociatedSetIntersectAndSub(t *testing.T) {
|
||||
bh3, blob3 := makeFakePackedBlob()
|
||||
bh4, blob4 := makeFakePackedBlob()
|
||||
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, saver))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, saver))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, restic.Blobs{blob3.Blob}, saver))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, restic.Blobs{blob4.Blob}, saver))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, saver))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, saver))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob3.PackID(), pack.Blobs{blob3.Blob}, saver))
|
||||
test.OK(t, mi.StorePack(context.TODO(), blob4.PackID(), pack.Blobs{blob4.Blob}, saver))
|
||||
test.OK(t, mi.Flush(context.TODO(), saver))
|
||||
|
||||
t.Run("Intersect", func(t *testing.T) {
|
||||
|
||||
@@ -73,7 +73,7 @@ func (idx *Index) addToPacks(id restic.ID) int {
|
||||
return len(idx.packs) - 1
|
||||
}
|
||||
|
||||
func (idx *Index) store(packIndex int, blob restic.Blob) {
|
||||
func (idx *Index) store(packIndex int, blob pack.Blob) {
|
||||
// assert that offset and length fit into uint32!
|
||||
if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 {
|
||||
panic("offset or length does not fit in uint32. You have packs > 4GB!")
|
||||
@@ -146,7 +146,7 @@ func (idx *Index) Preallocate(t restic.BlobType, numEntries int) {
|
||||
|
||||
// StorePack remembers the ids of all blobs of a given pack
|
||||
// in the index
|
||||
func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) {
|
||||
func (idx *Index) StorePack(id restic.ID, blobs pack.Blobs) {
|
||||
idx.m.Lock()
|
||||
defer idx.m.Unlock()
|
||||
|
||||
@@ -162,23 +162,24 @@ func (idx *Index) StorePack(id restic.ID, blobs restic.Blobs) {
|
||||
}
|
||||
}
|
||||
|
||||
func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) restic.PackedBlob {
|
||||
return restic.PackedBlob{
|
||||
Blob: restic.Blob{
|
||||
func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) *pack.PackedBlob {
|
||||
return &pack.PackedBlob{
|
||||
Pack: idx.packs[e.packIndex],
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: restic.BlobHandle{
|
||||
ID: e.id,
|
||||
Type: t},
|
||||
Type: t,
|
||||
},
|
||||
Length: uint(e.length),
|
||||
Offset: uint(e.offset),
|
||||
UncompressedLength: uint(e.uncompressedLength),
|
||||
},
|
||||
PackID: idx.packs[e.packIndex],
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup queries the index for the blob ID and returns all entries including
|
||||
// duplicates. Adds found entries to blobs and returns the result.
|
||||
func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic.PackedBlob {
|
||||
// duplicates. Adds found entries to pbs and returns the result.
|
||||
func (idx *Index) Lookup(bh restic.BlobHandle, pbs []*pack.PackedBlob) []*pack.PackedBlob {
|
||||
idx.m.RLock()
|
||||
defer idx.m.RUnlock()
|
||||
|
||||
@@ -215,8 +216,8 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found
|
||||
|
||||
// Values returns an iterator over all blobs known to the index. This blocks any
|
||||
// modification of the index.
|
||||
func (idx *Index) Values() iter.Seq[restic.PackedBlob] {
|
||||
return func(yield func(restic.PackedBlob) bool) {
|
||||
func (idx *Index) Values() iter.Seq[*pack.PackedBlob] {
|
||||
return func(yield func(*pack.PackedBlob) bool) {
|
||||
idx.m.RLock()
|
||||
defer idx.m.RUnlock()
|
||||
|
||||
@@ -231,17 +232,23 @@ func (idx *Index) Values() iter.Seq[restic.PackedBlob] {
|
||||
}
|
||||
}
|
||||
|
||||
// EachByPack returns a channel that yields all blobs known to the index
|
||||
// PackBlobs lists all blobs contained in a pack file according to the index.
|
||||
type PackBlobs struct {
|
||||
PackID restic.ID
|
||||
Blobs pack.Blobs
|
||||
}
|
||||
|
||||
// EachByPack returns a channel that yields all blobs known to the index,
|
||||
// grouped by packID but ignoring blobs with a packID in packPlacklist for
|
||||
// finalized indexes.
|
||||
// This filtering is used when rebuilding the index where we need to ignore packs
|
||||
// from the finalized index which have been re-read into a non-finalized index.
|
||||
// When the context is cancelled, the background goroutine
|
||||
// terminates. This blocks any modification of the index.
|
||||
func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan restic.PackBlobs {
|
||||
func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan PackBlobs {
|
||||
idx.m.RLock()
|
||||
|
||||
ch := make(chan restic.PackBlobs)
|
||||
ch := make(chan PackBlobs)
|
||||
|
||||
go func() {
|
||||
defer idx.m.RUnlock()
|
||||
@@ -262,7 +269,7 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-
|
||||
}
|
||||
|
||||
for packID, packByType := range byPack {
|
||||
var result restic.PackBlobs
|
||||
var result PackBlobs
|
||||
result.PackID = packID
|
||||
for typ, p := range packByType {
|
||||
for _, e := range p {
|
||||
@@ -482,7 +489,7 @@ func (idx *Index) merge(idx2 *Index) error {
|
||||
for e := range m.valuesWithID(e2.id) {
|
||||
b := idx.toPackedBlob(e, restic.BlobType(typ))
|
||||
b2 := idx2.toPackedBlob(e2, restic.BlobType(typ))
|
||||
if b == b2 {
|
||||
if b.Blob == b2.Blob && b.PackID() == b2.PackID() {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
@@ -519,7 +526,7 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, err error) {
|
||||
packID := idx.addToPacks(p.ID)
|
||||
|
||||
for _, blob := range p.Blobs {
|
||||
idx.store(packID, restic.Blob{
|
||||
idx.store(packID, pack.Blob{
|
||||
BlobHandle: restic.BlobHandle{
|
||||
Type: blob.Type,
|
||||
ID: blob.ID},
|
||||
@@ -550,7 +557,7 @@ func (idx *Index) Len(t restic.BlobType) uint {
|
||||
return idx.byType[t].len()
|
||||
}
|
||||
|
||||
func PackBlobsHash(pbs restic.PackBlobs) restic.ID {
|
||||
func PackBlobsHash(pbs PackBlobs) restic.ID {
|
||||
h := sha256.New()
|
||||
h.Write(pbs.PackID[:])
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ func TestIndexOversized(t *testing.T) {
|
||||
// Add blobs up to indexMaxBlobs + pack.MaxHeaderEntries - 1
|
||||
packID := idx.addToPacks(restic.NewRandomID())
|
||||
for i := uint(0); i < indexMaxBlobs+pack.MaxHeaderEntries-1; i++ {
|
||||
idx.store(packID, restic.Blob{
|
||||
idx.store(packID, pack.Blob{
|
||||
BlobHandle: restic.BlobHandle{
|
||||
Type: restic.DataBlob,
|
||||
ID: restic.NewRandomID(),
|
||||
@@ -27,7 +27,7 @@ func TestIndexOversized(t *testing.T) {
|
||||
rtest.Assert(t, !Oversized(idx), "index should not be considered oversized")
|
||||
|
||||
// Add one more blob to exceed the limit
|
||||
idx.store(packID, restic.Blob{
|
||||
idx.store(packID, pack.Blob{
|
||||
BlobHandle: restic.BlobHandle{
|
||||
Type: restic.DataBlob,
|
||||
ID: restic.NewRandomID(),
|
||||
|
||||
@@ -9,19 +9,20 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func TestIndexSerialize(t *testing.T) {
|
||||
tests := []restic.PackedBlob{}
|
||||
tests := []*pack.PackedBlob{}
|
||||
|
||||
idx := index.NewIndex()
|
||||
|
||||
// create 50 packs with 20 blobs each
|
||||
for i := 0; i < 50; i++ {
|
||||
packID := restic.NewRandomID()
|
||||
var blobs restic.Blobs
|
||||
var blobs pack.Blobs
|
||||
|
||||
pos := uint(0)
|
||||
for j := 0; j < 20; j++ {
|
||||
@@ -31,14 +32,14 @@ func TestIndexSerialize(t *testing.T) {
|
||||
// test a mix of compressed and uncompressed packs
|
||||
uncompressedLength = 2 * length
|
||||
}
|
||||
pb := restic.PackedBlob{
|
||||
Blob: restic.Blob{
|
||||
pb := &pack.PackedBlob{
|
||||
Pack: packID,
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Offset: pos,
|
||||
Length: length,
|
||||
UncompressedLength: uncompressedLength,
|
||||
},
|
||||
PackID: packID,
|
||||
}
|
||||
blobs = append(blobs, pb.Blob)
|
||||
tests = append(tests, pb)
|
||||
@@ -64,17 +65,17 @@ func TestIndexSerialize(t *testing.T) {
|
||||
rtest.OK(t, err)
|
||||
|
||||
for _, testBlob := range tests {
|
||||
list := idx.Lookup(testBlob.BlobHandle, nil)
|
||||
list := idx.Lookup(testBlob.Handle(), nil)
|
||||
if len(list) != 1 {
|
||||
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list)
|
||||
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list)
|
||||
}
|
||||
result := list[0]
|
||||
|
||||
rtest.Equals(t, testBlob, result)
|
||||
|
||||
list2 := idx2.Lookup(testBlob.BlobHandle, nil)
|
||||
list2 := idx2.Lookup(testBlob.Handle(), nil)
|
||||
if len(list2) != 1 {
|
||||
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list2), list2)
|
||||
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list2), list2)
|
||||
}
|
||||
result2 := list2[0]
|
||||
|
||||
@@ -82,21 +83,21 @@ func TestIndexSerialize(t *testing.T) {
|
||||
}
|
||||
|
||||
// add more blobs to idx
|
||||
newtests := []restic.PackedBlob{}
|
||||
newtests := []*pack.PackedBlob{}
|
||||
for i := 0; i < 10; i++ {
|
||||
packID := restic.NewRandomID()
|
||||
var blobs restic.Blobs
|
||||
var blobs pack.Blobs
|
||||
|
||||
pos := uint(0)
|
||||
for j := 0; j < 10; j++ {
|
||||
length := uint(i*100 + j)
|
||||
pb := restic.PackedBlob{
|
||||
Blob: restic.Blob{
|
||||
pb := &pack.PackedBlob{
|
||||
Pack: packID,
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Offset: pos,
|
||||
Length: length,
|
||||
},
|
||||
PackID: packID,
|
||||
}
|
||||
blobs = append(blobs, pb.Blob)
|
||||
newtests = append(newtests, pb)
|
||||
@@ -127,9 +128,9 @@ func TestIndexSerialize(t *testing.T) {
|
||||
|
||||
// all new blobs must be in the index
|
||||
for _, testBlob := range newtests {
|
||||
list := idx3.Lookup(testBlob.BlobHandle, nil)
|
||||
list := idx3.Lookup(testBlob.Handle(), nil)
|
||||
if len(list) != 1 {
|
||||
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.ID.Str(), len(list), list)
|
||||
t.Errorf("expected one result for blob %v, got %v: %v", testBlob.Handle().ID.String(), len(list), list)
|
||||
}
|
||||
|
||||
blob := list[0]
|
||||
@@ -145,12 +146,12 @@ func TestIndexSize(t *testing.T) {
|
||||
blobCount := 100
|
||||
for i := 0; i < packs; i++ {
|
||||
packID := restic.NewRandomID()
|
||||
var blobs restic.Blobs
|
||||
var blobs pack.Blobs
|
||||
|
||||
pos := uint(0)
|
||||
for j := 0; j < blobCount; j++ {
|
||||
length := uint(i*100 + j)
|
||||
blobs = append(blobs, restic.Blob{
|
||||
blobs = append(blobs, pack.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Offset: pos,
|
||||
Length: length,
|
||||
@@ -293,15 +294,15 @@ func TestIndexUnserialize(t *testing.T) {
|
||||
|
||||
t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob)
|
||||
|
||||
rtest.Equals(t, test.packID, blob.PackID)
|
||||
rtest.Equals(t, test.tpe, blob.Type)
|
||||
rtest.Equals(t, test.offset, blob.Offset)
|
||||
rtest.Equals(t, test.length, blob.Length)
|
||||
rtest.Equals(t, test.packID, blob.PackID())
|
||||
rtest.Equals(t, test.tpe, blob.Blob.Type)
|
||||
rtest.Equals(t, test.offset, blob.Blob.Offset)
|
||||
rtest.Equals(t, test.length, blob.Blob.Length)
|
||||
switch task.version {
|
||||
case 1:
|
||||
rtest.Equals(t, uint(0), blob.UncompressedLength)
|
||||
rtest.Equals(t, uint(0), blob.Blob.UncompressedLength)
|
||||
case 2:
|
||||
rtest.Equals(t, test.uncompressedLength, blob.UncompressedLength)
|
||||
rtest.Equals(t, test.uncompressedLength, blob.Blob.UncompressedLength)
|
||||
default:
|
||||
t.Fatal("Invalid index version")
|
||||
}
|
||||
@@ -313,20 +314,20 @@ func TestIndexUnserialize(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, blob := range blobs {
|
||||
b, ok := exampleLookupTest.blobs[blob.ID]
|
||||
b, ok := exampleLookupTest.blobs[blob.Handle().ID]
|
||||
if !ok {
|
||||
t.Errorf("unexpected blob %v found", blob.ID.Str())
|
||||
t.Errorf("unexpected blob %v found", blob.Handle().ID.String())
|
||||
}
|
||||
if blob.Type != b {
|
||||
t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type)
|
||||
if blob.Blob.Type != b {
|
||||
t.Errorf("unexpected type for blob %v: want %v, got %v", blob.Handle().ID.String(), b, blob.Blob.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
|
||||
func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []*pack.PackedBlob) {
|
||||
for pb := range idx.Values() {
|
||||
if pb.PackID.Equal(id) {
|
||||
if pb.PackID().Equal(id) {
|
||||
pbs = append(pbs, pb)
|
||||
}
|
||||
}
|
||||
@@ -401,7 +402,7 @@ func TestIndexPacks(t *testing.T) {
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
packID := restic.NewRandomID()
|
||||
idx.StorePack(packID, restic.Blobs{
|
||||
idx.StorePack(packID, pack.Blobs{
|
||||
{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Offset: 0,
|
||||
@@ -433,12 +434,12 @@ func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupB
|
||||
// create index with given number of pack files
|
||||
for i := 0; i < packfiles; i++ {
|
||||
packID := NewRandomTestID(rng)
|
||||
var blobs restic.Blobs
|
||||
var blobs pack.Blobs
|
||||
offset := 0
|
||||
for offset < maxPackSize {
|
||||
size := 2000 + rng.Intn(4*1024*1024)
|
||||
id := NewRandomTestID(rng)
|
||||
blobs = append(blobs, restic.Blob{
|
||||
blobs = append(blobs, pack.Blob{
|
||||
BlobHandle: restic.BlobHandle{
|
||||
Type: restic.DataBlob,
|
||||
ID: id,
|
||||
@@ -482,7 +483,7 @@ func BenchmarkIndexHasKnown(b *testing.B) {
|
||||
idx, _ := createRandomIndex(rand.New(rand.NewSource(0)), 200000)
|
||||
handles := make([]restic.BlobHandle, 0, 100000)
|
||||
for handle := range idx.Values() {
|
||||
handles = append(handles, handle.BlobHandle)
|
||||
handles = append(handles, handle.Handle())
|
||||
if len(handles) == cap(handles) {
|
||||
break
|
||||
}
|
||||
@@ -517,14 +518,14 @@ func BenchmarkIndexAllocParallel(b *testing.B) {
|
||||
}
|
||||
|
||||
func TestIndexHas(t *testing.T) {
|
||||
tests := []restic.PackedBlob{}
|
||||
tests := []*pack.PackedBlob{}
|
||||
|
||||
idx := index.NewIndex()
|
||||
|
||||
// create 50 packs with 20 blobs each
|
||||
for i := 0; i < 50; i++ {
|
||||
packID := restic.NewRandomID()
|
||||
var blobs restic.Blobs
|
||||
var blobs pack.Blobs
|
||||
|
||||
pos := uint(0)
|
||||
for j := 0; j < 20; j++ {
|
||||
@@ -534,14 +535,14 @@ func TestIndexHas(t *testing.T) {
|
||||
// test a mix of compressed and uncompressed packs
|
||||
uncompressedLength = 2 * length
|
||||
}
|
||||
pb := restic.PackedBlob{
|
||||
Blob: restic.Blob{
|
||||
pb := &pack.PackedBlob{
|
||||
Pack: packID,
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Offset: pos,
|
||||
Length: length,
|
||||
UncompressedLength: uncompressedLength,
|
||||
},
|
||||
PackID: packID,
|
||||
}
|
||||
blobs = append(blobs, pb.Blob)
|
||||
tests = append(tests, pb)
|
||||
@@ -551,11 +552,11 @@ func TestIndexHas(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, testBlob := range tests {
|
||||
rtest.Assert(t, idx.Has(testBlob.BlobHandle), "Index reports not having data blob added to it")
|
||||
rtest.Assert(t, idx.Has(testBlob.Handle()), "Index reports not having data blob added to it")
|
||||
}
|
||||
|
||||
rtest.Assert(t, !idx.Has(restic.NewRandomBlobHandle()), "Index reports having a data blob not added to it")
|
||||
rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob")
|
||||
rtest.Assert(t, !idx.Has(restic.BlobHandle{ID: tests[0].Handle().ID, Type: restic.TreeBlob}), "Index reports having a tree blob added to it with the same id as a data blob")
|
||||
}
|
||||
|
||||
func TestMixedEachByPack(t *testing.T) {
|
||||
@@ -566,7 +567,7 @@ func TestMixedEachByPack(t *testing.T) {
|
||||
for i := 0; i < 50; i++ {
|
||||
packID := restic.NewRandomID()
|
||||
expected[packID] = 1
|
||||
blobs := restic.Blobs{
|
||||
blobs := pack.Blobs{
|
||||
{
|
||||
BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()},
|
||||
Offset: 0,
|
||||
@@ -608,7 +609,7 @@ func TestEachByPackIgnoes(t *testing.T) {
|
||||
} else {
|
||||
expected[packID] = 1
|
||||
}
|
||||
blobs := restic.Blobs{
|
||||
blobs := pack.Blobs{
|
||||
{
|
||||
BlobHandle: restic.BlobHandle{Type: restic.DataBlob, ID: restic.NewRandomID()},
|
||||
Offset: 0,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
"golang.org/x/sync/errgroup"
|
||||
@@ -39,10 +40,11 @@ func (mi *MasterIndex) clearPendingBlobs() {
|
||||
}
|
||||
|
||||
// Lookup queries all known Indexes for the ID and returns all matches.
|
||||
func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) {
|
||||
func (mi *MasterIndex) Lookup(bh restic.BlobHandle) []*pack.PackedBlob {
|
||||
mi.idxMutex.RLock()
|
||||
defer mi.idxMutex.RUnlock()
|
||||
|
||||
var pbs []*pack.PackedBlob
|
||||
for _, idx := range mi.idx {
|
||||
pbs = idx.Lookup(bh, pbs)
|
||||
}
|
||||
@@ -145,12 +147,12 @@ func (mi *MasterIndex) Insert(idx *Index) {
|
||||
}
|
||||
|
||||
// StorePack remembers the id and pack in the index.
|
||||
func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs restic.Blobs, r restic.SaverUnpacked[restic.FileType]) error {
|
||||
func (mi *MasterIndex) StorePack(ctx context.Context, id restic.ID, blobs pack.Blobs, r restic.SaverUnpacked[restic.FileType]) error {
|
||||
mi.storePack(id, blobs)
|
||||
return mi.saveFullIndex(ctx, r)
|
||||
}
|
||||
|
||||
func (mi *MasterIndex) storePack(id restic.ID, blobs restic.Blobs) {
|
||||
func (mi *MasterIndex) storePack(id restic.ID, blobs pack.Blobs) {
|
||||
mi.idxMutex.Lock()
|
||||
defer mi.idxMutex.Unlock()
|
||||
|
||||
@@ -218,8 +220,8 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index {
|
||||
|
||||
// Values returns an iterator over all blobs known to the index. This blocks any
|
||||
// modification of the index.
|
||||
func (mi *MasterIndex) Values() iter.Seq[restic.PackedBlob] {
|
||||
return func(yield func(restic.PackedBlob) bool) {
|
||||
func (mi *MasterIndex) Values() iter.Seq[*pack.PackedBlob] {
|
||||
return func(yield func(*pack.PackedBlob) bool) {
|
||||
mi.idxMutex.RLock()
|
||||
defer mi.idxMutex.RUnlock()
|
||||
|
||||
@@ -663,13 +665,13 @@ func (mi *MasterIndex) saveFullIndex(ctx context.Context, r restic.SaverUnpacked
|
||||
}
|
||||
|
||||
// ListPacks returns the blobs of the specified pack files grouped by pack file.
|
||||
func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs {
|
||||
out := make(chan restic.PackBlobs)
|
||||
func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan PackBlobs {
|
||||
out := make(chan PackBlobs)
|
||||
go func() {
|
||||
defer close(out)
|
||||
// only resort a part of the index to keep the memory overhead bounded
|
||||
for i := byte(0); i < 16; i++ {
|
||||
packBlob := make(map[restic.ID]restic.Blobs)
|
||||
packBlob := make(map[restic.ID]pack.Blobs)
|
||||
for pack := range packs {
|
||||
if pack[0]&0xf == i {
|
||||
packBlob[pack] = nil
|
||||
@@ -682,8 +684,9 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i {
|
||||
packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob)
|
||||
if packs.Has(pb.PackID()) && pb.PackID()[0]&0xf == i {
|
||||
id := pb.PackID()
|
||||
packBlob[id] = append(packBlob[id], pb.Blob)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -692,7 +695,7 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
|
||||
// allow GC
|
||||
packBlob[packID] = nil
|
||||
select {
|
||||
case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}:
|
||||
case out <- PackBlobs{PackID: packID, Blobs: pbs}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/restic/restic/internal/data"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
@@ -24,18 +25,18 @@ func TestMasterIndex(t *testing.T) {
|
||||
bhInIdx2 := restic.NewRandomBlobHandle()
|
||||
bhInIdx12 := restic.BlobHandle{ID: restic.NewRandomID(), Type: restic.TreeBlob}
|
||||
|
||||
blob1 := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blob1 := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bhInIdx1,
|
||||
Length: uint(crypto.CiphertextLength(10)),
|
||||
Offset: 0,
|
||||
},
|
||||
}
|
||||
|
||||
blob2 := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blob2 := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bhInIdx2,
|
||||
Length: uint(crypto.CiphertextLength(100)),
|
||||
Offset: 10,
|
||||
@@ -43,9 +44,9 @@ func TestMasterIndex(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
blob12a := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blob12a := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bhInIdx12,
|
||||
Length: uint(crypto.CiphertextLength(123)),
|
||||
Offset: 110,
|
||||
@@ -53,9 +54,9 @@ func TestMasterIndex(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
blob12b := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blob12b := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bhInIdx12,
|
||||
Length: uint(crypto.CiphertextLength(123)),
|
||||
Offset: 50,
|
||||
@@ -64,12 +65,12 @@ func TestMasterIndex(t *testing.T) {
|
||||
}
|
||||
|
||||
idx1 := index.NewIndex()
|
||||
idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob})
|
||||
idx1.StorePack(blob12a.PackID, restic.Blobs{blob12a.Blob})
|
||||
idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob})
|
||||
idx1.StorePack(blob12a.PackID(), pack.Blobs{blob12a.Blob})
|
||||
|
||||
idx2 := index.NewIndex()
|
||||
idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob})
|
||||
idx2.StorePack(blob12b.PackID, restic.Blobs{blob12b.Blob})
|
||||
idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob})
|
||||
idx2.StorePack(blob12b.PackID(), pack.Blobs{blob12b.Blob})
|
||||
|
||||
mIdx := index.NewMasterIndex()
|
||||
mIdx.Insert(idx1)
|
||||
@@ -77,7 +78,7 @@ func TestMasterIndex(t *testing.T) {
|
||||
|
||||
// test idInIdx1
|
||||
blobs := mIdx.Lookup(bhInIdx1)
|
||||
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
|
||||
rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs)
|
||||
|
||||
size, found := mIdx.LookupSize(bhInIdx1)
|
||||
rtest.Equals(t, true, found)
|
||||
@@ -85,7 +86,7 @@ func TestMasterIndex(t *testing.T) {
|
||||
|
||||
// test idInIdx2
|
||||
blobs = mIdx.Lookup(bhInIdx2)
|
||||
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
|
||||
rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs)
|
||||
|
||||
size, found = mIdx.LookupSize(bhInIdx2)
|
||||
rtest.Equals(t, true, found)
|
||||
@@ -95,19 +96,20 @@ func TestMasterIndex(t *testing.T) {
|
||||
blobs = mIdx.Lookup(bhInIdx12)
|
||||
rtest.Equals(t, 2, len(blobs))
|
||||
|
||||
// test Lookup result for blob12a
|
||||
found = false
|
||||
if blobs[0] == blob12a || blobs[1] == blob12a {
|
||||
found = true
|
||||
containsPackedBlob := func(list []*pack.PackedBlob, want *pack.PackedBlob) bool {
|
||||
for _, b := range list {
|
||||
if b.PackID().Equal(want.PackID()) && b.Blob == want.Blob {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
rtest.Assert(t, found, "blob12a not found in result")
|
||||
|
||||
// test Lookup result for blob12a
|
||||
rtest.Assert(t, containsPackedBlob(blobs, blob12a), "blob12a not found in result")
|
||||
|
||||
// test Lookup result for blob12b
|
||||
found = false
|
||||
if blobs[0] == blob12b || blobs[1] == blob12b {
|
||||
found = true
|
||||
}
|
||||
rtest.Assert(t, found, "blob12a not found in result")
|
||||
rtest.Assert(t, containsPackedBlob(blobs, blob12b), "blob12b not found in result")
|
||||
|
||||
size, found = mIdx.LookupSize(bhInIdx12)
|
||||
rtest.Equals(t, true, found)
|
||||
@@ -135,7 +137,7 @@ func TestMasterIndexAddPending(t *testing.T) {
|
||||
// Test AddPending: try to add a blob that's already in an index (should return false)
|
||||
bhInIndex := restic.NewRandomBlobHandle()
|
||||
idx := index.NewIndex()
|
||||
idx.StorePack(restic.NewRandomID(), restic.Blobs{{
|
||||
idx.StorePack(restic.NewRandomID(), pack.Blobs{{
|
||||
BlobHandle: bhInIndex,
|
||||
Length: uint(crypto.CiphertextLength(50)),
|
||||
Offset: 0,
|
||||
@@ -173,14 +175,14 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) {
|
||||
|
||||
// Store the blob in a pack
|
||||
packID := restic.NewRandomID()
|
||||
blob := restic.Blob{
|
||||
blob := pack.Blob{
|
||||
BlobHandle: bhPending,
|
||||
Length: uint(crypto.CiphertextLength(75)),
|
||||
Offset: 0,
|
||||
UncompressedLength: 75,
|
||||
}
|
||||
saver := &noopSaver{}
|
||||
err := mIdx.StorePack(context.Background(), packID, restic.Blobs{blob}, saver)
|
||||
err := mIdx.StorePack(context.Background(), packID, pack.Blobs{blob}, saver)
|
||||
rtest.OK(t, err)
|
||||
|
||||
// Verify it is still found
|
||||
@@ -191,8 +193,8 @@ func TestMasterIndexStorePackRemovesPending(t *testing.T) {
|
||||
// Verify the blob can be found via Lookup from the index
|
||||
blobs := mIdx.Lookup(bhPending)
|
||||
rtest.Assert(t, len(blobs) > 0, "blob should be found in index after StorePack")
|
||||
rtest.Equals(t, packID, blobs[0].PackID)
|
||||
rtest.Equals(t, bhPending, blobs[0].BlobHandle)
|
||||
rtest.Equals(t, packID, blobs[0].PackID())
|
||||
rtest.Equals(t, bhPending, blobs[0].Handle())
|
||||
|
||||
// Test that adding the same blob as pending again fails (it's now in index)
|
||||
added = mIdx.AddPending(bhPending, 100)
|
||||
@@ -203,18 +205,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
||||
bhInIdx1 := restic.NewRandomBlobHandle()
|
||||
bhInIdx2 := restic.NewRandomBlobHandle()
|
||||
|
||||
blob1 := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blob1 := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bhInIdx1,
|
||||
Length: 10,
|
||||
Offset: 0,
|
||||
},
|
||||
}
|
||||
|
||||
blob2 := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blob2 := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bhInIdx2,
|
||||
Length: 100,
|
||||
Offset: 10,
|
||||
@@ -223,10 +225,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
||||
}
|
||||
|
||||
idx1 := index.NewIndex()
|
||||
idx1.StorePack(blob1.PackID, restic.Blobs{blob1.Blob})
|
||||
idx1.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob})
|
||||
|
||||
idx2 := index.NewIndex()
|
||||
idx2.StorePack(blob2.PackID, restic.Blobs{blob2.Blob})
|
||||
idx2.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob})
|
||||
|
||||
mIdx := index.NewMasterIndex()
|
||||
mIdx.Insert(idx1)
|
||||
@@ -246,18 +248,18 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
||||
rtest.Equals(t, 2, blobCount)
|
||||
|
||||
blobs := mIdx.Lookup(bhInIdx1)
|
||||
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
|
||||
rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs)
|
||||
|
||||
blobs = mIdx.Lookup(bhInIdx2)
|
||||
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
|
||||
rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs)
|
||||
|
||||
blobs = mIdx.Lookup(restic.NewRandomBlobHandle())
|
||||
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
|
||||
|
||||
// merge another index containing identical blobs
|
||||
idx3 := index.NewIndex()
|
||||
idx3.StorePack(blob1.PackID, restic.Blobs{blob1.Blob})
|
||||
idx3.StorePack(blob2.PackID, restic.Blobs{blob2.Blob})
|
||||
idx3.StorePack(blob1.PackID(), pack.Blobs{blob1.Blob})
|
||||
idx3.StorePack(blob2.PackID(), pack.Blobs{blob2.Blob})
|
||||
|
||||
mIdx.Insert(idx3)
|
||||
finalIndexes, idxCount, newIDs := index.TestMergeIndex(t, mIdx)
|
||||
@@ -268,10 +270,10 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
||||
|
||||
// Index should have same entries as before!
|
||||
blobs = mIdx.Lookup(bhInIdx1)
|
||||
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
|
||||
rtest.Equals(t, []*pack.PackedBlob{blob1}, blobs)
|
||||
|
||||
blobs = mIdx.Lookup(bhInIdx2)
|
||||
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
|
||||
rtest.Equals(t, []*pack.PackedBlob{blob2}, blobs)
|
||||
|
||||
blobCount = 0
|
||||
for range mIdx.Values() {
|
||||
@@ -448,9 +450,9 @@ func testIndexSave(t *testing.T, version uint) {
|
||||
|
||||
idx := index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
blobs := make(map[restic.PackedBlob]struct{})
|
||||
blobs := make(map[pack.PackedBlob]struct{})
|
||||
for pb := range idx.Values() {
|
||||
blobs[pb] = struct{}{}
|
||||
blobs[*pb] = struct{}{}
|
||||
}
|
||||
|
||||
rtest.OK(t, test.saver(idx, unpacked))
|
||||
@@ -458,8 +460,8 @@ func testIndexSave(t *testing.T, version uint) {
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
|
||||
for pb := range idx.Values() {
|
||||
if _, ok := blobs[pb]; ok {
|
||||
delete(blobs, pb)
|
||||
if _, ok := blobs[*pb]; ok {
|
||||
delete(blobs, *pb)
|
||||
} else {
|
||||
t.Fatalf("unexpected blobs %v", pb)
|
||||
}
|
||||
@@ -481,9 +483,9 @@ func testIndexSavePartial(t *testing.T, version uint) {
|
||||
// capture blob list before adding fourth snapshot
|
||||
idx := index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
blobs := make(map[restic.PackedBlob]struct{})
|
||||
blobs := make(map[pack.PackedBlob]struct{})
|
||||
for pb := range idx.Values() {
|
||||
blobs[pb] = struct{}{}
|
||||
blobs[*pb] = struct{}{}
|
||||
}
|
||||
|
||||
// add+remove new snapshot and track its pack files
|
||||
@@ -502,8 +504,8 @@ func testIndexSavePartial(t *testing.T, version uint) {
|
||||
idx = index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
for pb := range idx.Values() {
|
||||
if _, ok := blobs[pb]; ok {
|
||||
delete(blobs, pb)
|
||||
if _, ok := blobs[*pb]; ok {
|
||||
delete(blobs, *pb)
|
||||
} else {
|
||||
t.Fatalf("unexpected blobs %v", pb)
|
||||
}
|
||||
@@ -516,7 +518,7 @@ func testIndexSavePartial(t *testing.T, version uint) {
|
||||
checker.TestCheckRepo(t, repo)
|
||||
}
|
||||
|
||||
func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[restic.PackedBlob]struct{} {
|
||||
func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, master *index.MasterIndex, indexCount int) map[pack.PackedBlob]struct{} {
|
||||
p := progress.NewCounter(0, 0, nil)
|
||||
rtest.OK(t, master.Load(context.TODO(), repo, p, nil))
|
||||
v, max := p.Get()
|
||||
@@ -525,10 +527,10 @@ func loadIndexAndCollectBlobs(t *testing.T, repo restic.ListerLoaderUnpacked, ma
|
||||
return collectBlobs(master)
|
||||
}
|
||||
|
||||
func collectBlobs(master *index.MasterIndex) map[restic.PackedBlob]struct{} {
|
||||
s := make(map[restic.PackedBlob]struct{})
|
||||
func collectBlobs(master *index.MasterIndex) map[pack.PackedBlob]struct{} {
|
||||
s := make(map[pack.PackedBlob]struct{})
|
||||
for pb := range master.Values() {
|
||||
s[pb] = struct{}{}
|
||||
s[*pb] = struct{}{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -588,17 +590,17 @@ func TestRewriteOversizedIndex(t *testing.T) {
|
||||
return idx.Len(restic.DataBlob) > 2*fullIndexCount
|
||||
}
|
||||
|
||||
var blobs restic.Blobs
|
||||
var blobs pack.Blobs
|
||||
|
||||
// build oversized index
|
||||
idx := index.NewIndex()
|
||||
numPacks := 5
|
||||
for p := 0; p < numPacks; p++ {
|
||||
packID := restic.NewRandomID()
|
||||
packBlobs := make(restic.Blobs, 0, fullIndexCount)
|
||||
packBlobs := make(pack.Blobs, 0, fullIndexCount)
|
||||
|
||||
for i := 0; i < fullIndexCount; i++ {
|
||||
blob := restic.Blob{
|
||||
blob := pack.Blob{
|
||||
BlobHandle: restic.BlobHandle{
|
||||
Type: restic.DataBlob,
|
||||
ID: restic.NewRandomID(),
|
||||
@@ -645,17 +647,17 @@ func TestRewriteSplitPacks(t *testing.T) {
|
||||
bh2 := restic.NewRandomBlobHandle()
|
||||
bhOther := restic.NewRandomBlobHandle()
|
||||
|
||||
blob1 := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blob1 := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bh1,
|
||||
Length: uint(crypto.CiphertextLength(10)),
|
||||
Offset: 0,
|
||||
},
|
||||
}
|
||||
blob2 := restic.PackedBlob{
|
||||
PackID: blob1.PackID,
|
||||
Blob: restic.Blob{
|
||||
blob2 := &pack.PackedBlob{
|
||||
Pack: blob1.PackID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bh2,
|
||||
Length: uint(crypto.CiphertextLength(100)),
|
||||
Offset: 10,
|
||||
@@ -663,9 +665,9 @@ func TestRewriteSplitPacks(t *testing.T) {
|
||||
},
|
||||
}
|
||||
// used to force index repacking
|
||||
blobOther := restic.PackedBlob{
|
||||
PackID: restic.NewRandomID(),
|
||||
Blob: restic.Blob{
|
||||
blobOther := &pack.PackedBlob{
|
||||
Pack: restic.NewRandomID(),
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: bhOther,
|
||||
Length: uint(crypto.CiphertextLength(100)),
|
||||
Offset: 10,
|
||||
@@ -673,25 +675,25 @@ func TestRewriteSplitPacks(t *testing.T) {
|
||||
}
|
||||
|
||||
mi := index.NewMasterIndex()
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID, restic.Blobs{blob1.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blob1.PackID(), pack.Blobs{blob1.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID, restic.Blobs{blob2.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID, restic.Blobs{blobOther.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blob2.PackID(), pack.Blobs{blob2.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), blobOther.PackID(), pack.Blobs{blobOther.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
|
||||
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID), nil, nil, index.MasterIndexRewriteOpts{}))
|
||||
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID()), nil, nil, index.MasterIndexRewriteOpts{}))
|
||||
|
||||
mi = index.NewMasterIndex()
|
||||
rtest.OK(t, mi.Load(context.TODO(), repo, nil, nil))
|
||||
|
||||
// test that all blobs are still in the index
|
||||
for _, blob := range []restic.PackedBlob{blob1, blob2} {
|
||||
blobs := mi.Lookup(blob.BlobHandle)
|
||||
rtest.Equals(t, []restic.PackedBlob{blob}, blobs)
|
||||
for _, blob := range []*pack.PackedBlob{blob1, blob2} {
|
||||
blobs := mi.Lookup(blob.Handle())
|
||||
rtest.Equals(t, []*pack.PackedBlob{blob}, blobs)
|
||||
}
|
||||
|
||||
blobs := mi.Lookup(blobOther.BlobHandle)
|
||||
blobs := mi.Lookup(blobOther.Handle())
|
||||
rtest.Equals(t, nil, blobs)
|
||||
}
|
||||
|
||||
@@ -713,17 +715,17 @@ func TestRewriteFullPacks(t *testing.T) {
|
||||
packA := restic.NewRandomID()
|
||||
packB := restic.NewRandomID()
|
||||
|
||||
blobA := restic.PackedBlob{
|
||||
PackID: packA,
|
||||
Blob: restic.Blob{
|
||||
blobA := &pack.PackedBlob{
|
||||
Pack: packA,
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Length: uint(crypto.CiphertextLength(10)),
|
||||
Offset: 0,
|
||||
},
|
||||
}
|
||||
blobB := restic.PackedBlob{
|
||||
PackID: packB,
|
||||
Blob: restic.Blob{
|
||||
blobB := &pack.PackedBlob{
|
||||
Pack: packB,
|
||||
Blob: pack.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Length: uint(crypto.CiphertextLength(50)),
|
||||
Offset: 0,
|
||||
@@ -731,11 +733,11 @@ func TestRewriteFullPacks(t *testing.T) {
|
||||
}
|
||||
|
||||
mi := index.NewMasterIndex()
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packA, pack.Blobs{blobA.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packB, pack.Blobs{blobB.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
|
||||
indexIDs := mi.IDs()
|
||||
@@ -750,6 +752,6 @@ func TestRewriteFullPacks(t *testing.T) {
|
||||
rtest.Equals(t, 2, len(afterRewrite))
|
||||
rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
|
||||
|
||||
rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
|
||||
rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
|
||||
rtest.Equals(t, []*pack.PackedBlob{blobA}, mi2.Lookup(blobA.Handle()))
|
||||
rtest.Equals(t, []*pack.PackedBlob{blobB}, mi2.Lookup(blobB.Handle()))
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.Load
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if !yield(IndexBlob{Handle: blob.BlobHandle}) {
|
||||
if !yield(IndexBlob{Handle: blob.Handle()}) {
|
||||
return stopIteration
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// BlobsInPack returns index entries for blobs stored in packID, sorted by offset.
|
||||
func BlobsInPack(repo *Repository, packID restic.ID) restic.Blobs {
|
||||
var blobs restic.Blobs
|
||||
func BlobsInPack(repo *Repository, packID restic.ID) pack.Blobs {
|
||||
var blobs pack.Blobs
|
||||
for pb := range repo.idx.Values() {
|
||||
if pb.PackID.Equal(packID) {
|
||||
if pb.PackID().Equal(packID) {
|
||||
blobs = append(blobs, pb.Blob)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Blob is one part of a file or a tree with pack layout information.
|
||||
type Blob struct {
|
||||
restic.BlobHandle
|
||||
Length uint
|
||||
Offset uint
|
||||
UncompressedLength uint
|
||||
}
|
||||
|
||||
func (b Blob) String() string {
|
||||
return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v, uncompressed length %v>",
|
||||
b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength)
|
||||
}
|
||||
|
||||
func (b Blob) DataLength() uint {
|
||||
if b.UncompressedLength != 0 {
|
||||
return b.UncompressedLength
|
||||
}
|
||||
return uint(crypto.PlaintextLength(int(b.Length)))
|
||||
}
|
||||
|
||||
func (b Blob) IsCompressed() bool {
|
||||
return b.UncompressedLength != 0
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// Blobs is a list of blobs with pack layout information (offset, length, ...).
|
||||
type Blobs []Blob
|
||||
|
||||
func (b Blobs) Sort() {
|
||||
slices.SortFunc(b, func(a, b Blob) int {
|
||||
return cmp.Compare(a.Offset, b.Offset)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package pack
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func TestBlobsSort(t *testing.T) {
|
||||
blobs := Blobs{
|
||||
{Offset: 100},
|
||||
{Offset: 0},
|
||||
{Offset: 50},
|
||||
}
|
||||
blobs.Sort()
|
||||
rtest.Equals(t, uint(0), blobs[0].Offset)
|
||||
rtest.Equals(t, uint(50), blobs[1].Offset)
|
||||
rtest.Equals(t, uint(100), blobs[2].Offset)
|
||||
}
|
||||
|
||||
func TestBlobsSortNilSlice(t *testing.T) {
|
||||
var blobs Blobs
|
||||
blobs.Sort()
|
||||
}
|
||||
@@ -21,7 +21,7 @@ var ErrBroken = errors.New("packer cannot be used after a write error")
|
||||
|
||||
// Packer is used to create a new Pack.
|
||||
type Packer struct {
|
||||
blobs restic.Blobs
|
||||
blobs []Blob
|
||||
|
||||
bytes uint
|
||||
k *crypto.Key
|
||||
@@ -56,7 +56,7 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte, uncompressedL
|
||||
return n, p.err
|
||||
}
|
||||
|
||||
c := restic.Blob{
|
||||
c := Blob{
|
||||
BlobHandle: restic.BlobHandle{Type: t, ID: id},
|
||||
Length: uint(n),
|
||||
Offset: p.bytes,
|
||||
@@ -129,7 +129,7 @@ func (p *Packer) Finalize() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyHeader(k *crypto.Key, header []byte, expected restic.Blobs) error {
|
||||
func verifyHeader(k *crypto.Key, header []byte, expected []Blob) error {
|
||||
// do not offer a way to skip the pack header verification, as pack headers are usually small enough
|
||||
// to not result in a significant performance impact
|
||||
|
||||
@@ -157,7 +157,7 @@ func (p *Packer) HeaderOverhead() int {
|
||||
}
|
||||
|
||||
// makeHeader constructs the header for p.
|
||||
func makeHeader(blobs restic.Blobs) ([]byte, error) {
|
||||
func makeHeader(blobs []Blob) ([]byte, error) {
|
||||
buf := make([]byte, 0, len(blobs)*int(entrySize))
|
||||
|
||||
for _, b := range blobs {
|
||||
@@ -232,7 +232,7 @@ func (p *Packer) HeaderFull() bool {
|
||||
}
|
||||
|
||||
// Blobs returns the slice of blobs that have been written.
|
||||
func (p *Packer) Blobs() restic.Blobs {
|
||||
func (p *Packer) Blobs() Blobs {
|
||||
p.m.Lock()
|
||||
defer p.m.Unlock()
|
||||
|
||||
@@ -348,7 +348,7 @@ func (e InvalidFileError) Error() string {
|
||||
|
||||
// List returns the list of entries found in a pack file and the length of the
|
||||
// header (including header size and crypto overhead)
|
||||
func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrSize uint32, err error) {
|
||||
func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries Blobs, hdrSize uint32, err error) {
|
||||
buf, err := readHeader(rd, size)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
@@ -367,7 +367,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS
|
||||
}
|
||||
|
||||
// might over allocate a bit if all blobs have EntrySize but only by a few percent
|
||||
entries = make(restic.Blobs, 0, uint(len(buf))/plainEntrySize)
|
||||
entries = make(Blobs, 0, uint(len(buf))/plainEntrySize)
|
||||
|
||||
pos := uint(0)
|
||||
for len(buf) > 0 {
|
||||
@@ -385,7 +385,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries restic.Blobs, hdrS
|
||||
return entries, hdrSize, nil
|
||||
}
|
||||
|
||||
func parseHeaderEntry(p []byte) (b restic.Blob, size uint, err error) {
|
||||
func parseHeaderEntry(p []byte) (b Blob, size uint, err error) {
|
||||
l := uint(len(p))
|
||||
size = plainEntrySize
|
||||
if l < plainEntrySize {
|
||||
@@ -427,7 +427,7 @@ func CalculateEntrySize(compressed bool) int {
|
||||
return int(plainEntrySize)
|
||||
}
|
||||
|
||||
func CalculateHeaderSize(blobs restic.Blobs) int {
|
||||
func CalculateHeaderSize(blobs Blobs) int {
|
||||
size := headerSize
|
||||
for _, blob := range blobs {
|
||||
size += CalculateEntrySize(blob.IsCompressed())
|
||||
@@ -439,10 +439,10 @@ func CalculateHeaderSize(blobs restic.Blobs) int {
|
||||
// If onlyHdr is set to true, only the size of the header is returned
|
||||
// Note that this function only gives correct sizes, if there are no
|
||||
// duplicates in the index.
|
||||
func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) {
|
||||
func Size(ctx context.Context, idx restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) {
|
||||
packSize := make(map[restic.ID]int64)
|
||||
|
||||
err := mi.ListBlobs(ctx, func(blob restic.PackBlob) {
|
||||
err := idx.ListBlobs(ctx, func(blob restic.PackBlob) {
|
||||
packID := blob.PackID()
|
||||
size, ok := packSize[packID]
|
||||
if !ok {
|
||||
|
||||
@@ -182,7 +182,7 @@ func TestReadRecords(t *testing.T) {
|
||||
func TestUnpackedVerification(t *testing.T) {
|
||||
// create random keys
|
||||
k := crypto.NewRandomKey()
|
||||
blobs := restic.Blobs{
|
||||
blobs := []Blob{
|
||||
{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Length: 42,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package pack
|
||||
|
||||
import "github.com/restic/restic/internal/restic"
|
||||
|
||||
// PackedBlob is one index entry for a blob in a pack (may be duplicate across indexes).
|
||||
type PackedBlob struct {
|
||||
Pack restic.ID
|
||||
Blob Blob
|
||||
}
|
||||
|
||||
func (pb *PackedBlob) PackID() restic.ID { return pb.Pack }
|
||||
|
||||
func (pb *PackedBlob) Handle() restic.BlobHandle { return pb.Blob.BlobHandle }
|
||||
|
||||
func (pb *PackedBlob) CiphertextLength() uint { return pb.Blob.Length }
|
||||
|
||||
func (pb *PackedBlob) PlaintextLength() uint { return pb.Blob.DataLength() }
|
||||
|
||||
func (pb *PackedBlob) IsCompressed() bool { return pb.Blob.IsCompressed() }
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/feature"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
|
||||
@@ -80,11 +82,11 @@ func repack(
|
||||
}
|
||||
|
||||
var keepMutex sync.Mutex
|
||||
downloadQueue := make(chan restic.PackBlobs)
|
||||
downloadQueue := make(chan index.PackBlobs)
|
||||
wg.Go(func() error {
|
||||
defer close(downloadQueue)
|
||||
for pbs := range repo.listPacksFromIndex(wgCtx, packs) {
|
||||
var packBlobs restic.Blobs
|
||||
var packBlobs pack.Blobs
|
||||
keepMutex.Lock()
|
||||
// filter out unnecessary blobs
|
||||
for _, entry := range pbs.Blobs {
|
||||
@@ -96,7 +98,7 @@ func repack(
|
||||
keepMutex.Unlock()
|
||||
|
||||
select {
|
||||
case downloadQueue <- restic.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}:
|
||||
case downloadQueue <- index.PackBlobs{PackID: pbs.PackID, Blobs: packBlobs}:
|
||||
case <-wgCtx.Done():
|
||||
return wgCtx.Err()
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"slices"
|
||||
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
@@ -71,8 +72,8 @@ func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printe
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]restic.Blobs, error) {
|
||||
packToBlobs := make(map[restic.ID]restic.Blobs)
|
||||
func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSet) (map[restic.ID]pack.Blobs, error) {
|
||||
packToBlobs := make(map[restic.ID]pack.Blobs)
|
||||
|
||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||
if ids.Has(id) {
|
||||
@@ -90,7 +91,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
|
||||
return packToBlobs, nil
|
||||
}
|
||||
|
||||
func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs restic.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error {
|
||||
func reuploadBlobsFromPack(ctx context.Context, repo *Repository, packID restic.ID, blobs pack.Blobs, printer progress.Printer, uploader restic.BlobSaverWithAsync) error {
|
||||
err := repo.loadBlobsFromPack(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
|
||||
if err != nil {
|
||||
printer.E("failed to load blob %v: %v", blob.ID, err)
|
||||
|
||||
@@ -213,7 +213,7 @@ type haver interface {
|
||||
}
|
||||
|
||||
// sortCachedPacksFirst moves all cached pack files to the front of blobs.
|
||||
func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) {
|
||||
func sortCachedPacksFirst(cache haver, blobs []*pack.PackedBlob) {
|
||||
if cache == nil {
|
||||
return
|
||||
}
|
||||
@@ -224,10 +224,10 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) {
|
||||
}
|
||||
|
||||
cached := blobs[:0]
|
||||
noncached := make([]restic.PackedBlob, 0, len(blobs)/2)
|
||||
noncached := make([]*pack.PackedBlob, 0, len(blobs)/2)
|
||||
|
||||
for _, blob := range blobs {
|
||||
if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) {
|
||||
if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID().String()}) {
|
||||
cached = append(cached, blob)
|
||||
continue
|
||||
}
|
||||
@@ -256,7 +256,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
|
||||
if err != nil {
|
||||
if r.cache != nil {
|
||||
for _, blob := range blobs {
|
||||
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()}
|
||||
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()}
|
||||
// ignore errors as there's not much we can do here
|
||||
_ = r.cache.Forget(h)
|
||||
}
|
||||
@@ -267,28 +267,28 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
|
||||
return buf, err
|
||||
}
|
||||
|
||||
func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, buf []byte) ([]byte, error) {
|
||||
func (r *Repository) loadBlob(ctx context.Context, blobs []*pack.PackedBlob, buf []byte) ([]byte, error) {
|
||||
var lastError error
|
||||
for _, blob := range blobs {
|
||||
debug.Log("blob %v found: %v", blob.BlobHandle, blob)
|
||||
debug.Log("blob %v found: %v", blob.Handle(), blob)
|
||||
// load blob from pack
|
||||
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()}
|
||||
h := backend.Handle{Type: restic.PackFile, Name: blob.PackID().String(), IsMetadata: blob.Blob.Type.IsMetadata()}
|
||||
|
||||
switch {
|
||||
case cap(buf) < int(blob.Length):
|
||||
buf = make([]byte, blob.Length)
|
||||
case len(buf) != int(blob.Length):
|
||||
buf = buf[:blob.Length]
|
||||
case cap(buf) < int(blob.Blob.Length):
|
||||
buf = make([]byte, blob.Blob.Length)
|
||||
case len(buf) != int(blob.Blob.Length):
|
||||
buf = buf[:blob.Blob.Length]
|
||||
}
|
||||
|
||||
_, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf)
|
||||
_, err := backend.ReadAt(ctx, r.be, h, int64(blob.Blob.Offset), buf)
|
||||
if err != nil {
|
||||
debug.Log("error loading blob %v: %v", blob, err)
|
||||
lastError = err
|
||||
continue
|
||||
}
|
||||
|
||||
it := newPackBlobIterator(blob.PackID, newByteReader(buf), blob.Offset, restic.Blobs{blob.Blob}, r.key, r.getZstdDecoder())
|
||||
it := newPackBlobIterator(blob.PackID(), newByteReader(buf), blob.Blob.Offset, pack.Blobs{blob.Blob}, r.key, r.getZstdDecoder())
|
||||
pbv, err := it.Next()
|
||||
|
||||
if err == nil {
|
||||
@@ -314,7 +314,7 @@ func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, bu
|
||||
return nil, lastError
|
||||
}
|
||||
|
||||
return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs))
|
||||
return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].Handle(), len(blobs))
|
||||
}
|
||||
|
||||
func (r *Repository) getZstdEncoder() *zstd.Encoder {
|
||||
@@ -673,8 +673,8 @@ func (r *Repository) Connections() uint {
|
||||
func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackBlob {
|
||||
entries := r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id})
|
||||
out := make([]restic.PackBlob, len(entries))
|
||||
for i, pb := range entries {
|
||||
out[i] = restic.AsPackBlob(pb)
|
||||
for i, e := range entries {
|
||||
out[i] = e
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -691,13 +691,13 @@ func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackBlob)) er
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
fn(restic.AsPackBlob(blob))
|
||||
fn(blob)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// listPacksFromIndex returns index entries for the given packs, grouped by pack file.
|
||||
func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs {
|
||||
func (r *Repository) listPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan index.PackBlobs {
|
||||
return r.idx.ListPacks(ctx, packs)
|
||||
}
|
||||
|
||||
@@ -964,7 +964,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
|
||||
}
|
||||
|
||||
// listPack returns blob entries from the pack file header including offsets.
|
||||
func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
|
||||
func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (pack.Blobs, error) {
|
||||
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
|
||||
|
||||
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
||||
@@ -977,7 +977,7 @@ func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (re
|
||||
// retry on error
|
||||
entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
||||
}
|
||||
return entries, err
|
||||
return pack.Blobs(entries), err
|
||||
}
|
||||
|
||||
// ListPackHandles returns the blob handles stored in the pack file header.
|
||||
@@ -1074,12 +1074,12 @@ func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, ha
|
||||
return r.loadBlobsFromPack(ctx, packID, blobs, handleBlobFn)
|
||||
}
|
||||
|
||||
func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (restic.Blobs, error) {
|
||||
blobs := make(restic.Blobs, 0, len(handles))
|
||||
func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle) (pack.Blobs, error) {
|
||||
blobs := make(pack.Blobs, 0, len(handles))
|
||||
for _, h := range handles {
|
||||
found := false
|
||||
for _, pb := range r.idx.Lookup(h) {
|
||||
if pb.PackID.Equal(packID) {
|
||||
if pb.PackID().Equal(packID) {
|
||||
blobs = append(blobs, pb.Blob)
|
||||
found = true
|
||||
break
|
||||
@@ -1092,11 +1092,11 @@ func (r *Repository) blobsInPack(packID restic.ID, handles []restic.BlobHandle)
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
|
||||
func (r *Repository) loadBlobsFromPack(ctx context.Context, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
|
||||
return streamPack(ctx, r.be.Load, r.LoadBlob, r.getZstdDecoder(), r.key, packID, blobs, handleBlobFn)
|
||||
}
|
||||
|
||||
func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
|
||||
func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
|
||||
if len(blobs) == 0 {
|
||||
// nothing to do
|
||||
return nil
|
||||
@@ -1139,7 +1139,7 @@ func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn
|
||||
return streamPackPart(ctx, beLoad, loadBlobFn, dec, key, packID, blobs[lowerIdx:], handleBlobFn)
|
||||
}
|
||||
|
||||
func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs restic.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
|
||||
func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs pack.Blobs, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
|
||||
h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: blobs[0].Type.IsMetadata()}
|
||||
|
||||
dataStart := blobs[0].Offset
|
||||
@@ -1249,7 +1249,7 @@ type packBlobIterator struct {
|
||||
rd discardReader
|
||||
currentOffset uint
|
||||
|
||||
blobs restic.Blobs
|
||||
blobs pack.Blobs
|
||||
key *crypto.Key
|
||||
dec *zstd.Decoder
|
||||
|
||||
@@ -1265,7 +1265,7 @@ type packBlobValue struct {
|
||||
var errPackEOF = errors.New("reached EOF of pack file")
|
||||
|
||||
func newPackBlobIterator(packID restic.ID, rd discardReader, currentOffset uint,
|
||||
blobs restic.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator {
|
||||
blobs pack.Blobs, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator {
|
||||
return &packBlobIterator{
|
||||
packID: packID,
|
||||
rd: rd,
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
@@ -27,7 +28,7 @@ func (c mapcache) Has(h backend.Handle) bool { return c[h] }
|
||||
|
||||
func TestSortCachedPacksFirst(t *testing.T) {
|
||||
var (
|
||||
blobs, sorted [100]restic.PackedBlob
|
||||
blobs, sorted [100]*pack.PackedBlob
|
||||
|
||||
cache = make(mapcache)
|
||||
r = rand.New(rand.NewSource(1261))
|
||||
@@ -36,7 +37,7 @@ func TestSortCachedPacksFirst(t *testing.T) {
|
||||
for i := 0; i < len(blobs); i++ {
|
||||
var id restic.ID
|
||||
r.Read(id[:])
|
||||
blobs[i] = restic.PackedBlob{PackID: id}
|
||||
blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}}
|
||||
|
||||
if i%3 == 0 {
|
||||
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
|
||||
@@ -46,8 +47,8 @@ func TestSortCachedPacksFirst(t *testing.T) {
|
||||
|
||||
copy(sorted[:], blobs[:])
|
||||
sort.SliceStable(sorted[:], func(i, j int) bool {
|
||||
hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()}
|
||||
hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()}
|
||||
hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID().String()}
|
||||
hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID().String()}
|
||||
return cache.Has(hi) && !cache.Has(hj)
|
||||
})
|
||||
|
||||
@@ -59,7 +60,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
|
||||
const nblobs = 512 // Corresponds to a file of ca. 2GB.
|
||||
|
||||
var (
|
||||
blobs [nblobs]restic.PackedBlob
|
||||
blobs [nblobs]*pack.PackedBlob
|
||||
cache = make(mapcache)
|
||||
r = rand.New(rand.NewSource(1261))
|
||||
)
|
||||
@@ -67,7 +68,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
|
||||
for i := 0; i < nblobs; i++ {
|
||||
var id restic.ID
|
||||
r.Read(id[:])
|
||||
blobs[i] = restic.PackedBlob{PackID: id}
|
||||
blobs[i] = &pack.PackedBlob{Pack: id, Blob: pack.Blob{}}
|
||||
|
||||
if i%3 == 0 {
|
||||
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
|
||||
@@ -75,7 +76,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
var cpy [nblobs]restic.PackedBlob
|
||||
var cpy [nblobs]*pack.PackedBlob
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
|
||||
@@ -96,7 +97,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
|
||||
idx := index.NewIndex()
|
||||
|
||||
for i := 0; i < 5000; i++ {
|
||||
idx.StorePack(restic.NewRandomID(), restic.Blobs{
|
||||
idx.StorePack(restic.NewRandomID(), pack.Blobs{
|
||||
{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Length: 1234,
|
||||
@@ -133,7 +134,7 @@ func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*
|
||||
}
|
||||
|
||||
// buildPackfileWithoutHeader returns a manually built pack file without a header.
|
||||
func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs restic.Blobs, packfile []byte) {
|
||||
func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs pack.Blobs, packfile []byte) {
|
||||
opts := []zstd.EOption{
|
||||
// Set the compression level configured.
|
||||
zstd.WithEncoderLevel(zstd.SpeedDefault),
|
||||
@@ -173,7 +174,7 @@ func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool)
|
||||
|
||||
ciphertextLength := after - before
|
||||
|
||||
blobs = append(blobs, restic.Blob{
|
||||
blobs = append(blobs, pack.Blob{
|
||||
BlobHandle: restic.BlobHandle{
|
||||
Type: restic.DataBlob,
|
||||
ID: id,
|
||||
@@ -280,19 +281,19 @@ func testStreamPack(t *testing.T, version uint) {
|
||||
// first, test regular usage
|
||||
t.Run("regular", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
blobs restic.Blobs
|
||||
blobs pack.Blobs
|
||||
calls int
|
||||
shortFirstLoad bool
|
||||
}{
|
||||
{packfileBlobs[1:2], 1, false},
|
||||
{packfileBlobs[2:5], 1, false},
|
||||
{packfileBlobs[2:8], 1, false},
|
||||
{restic.Blobs{
|
||||
{pack.Blobs{
|
||||
packfileBlobs[0],
|
||||
packfileBlobs[4],
|
||||
packfileBlobs[2],
|
||||
}, 1, false},
|
||||
{restic.Blobs{
|
||||
{pack.Blobs{
|
||||
packfileBlobs[0],
|
||||
packfileBlobs[len(packfileBlobs)-1],
|
||||
}, 2, false},
|
||||
@@ -341,12 +342,12 @@ func testStreamPack(t *testing.T, version uint) {
|
||||
// next, test invalid uses, which should return an error
|
||||
t.Run("invalid", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
blobs restic.Blobs
|
||||
blobs pack.Blobs
|
||||
err string
|
||||
}{
|
||||
{
|
||||
// pass one blob several times
|
||||
blobs: restic.Blobs{
|
||||
blobs: pack.Blobs{
|
||||
packfileBlobs[3],
|
||||
packfileBlobs[8],
|
||||
packfileBlobs[3],
|
||||
@@ -357,7 +358,7 @@ func testStreamPack(t *testing.T, version uint) {
|
||||
|
||||
{
|
||||
// pass something that's not a valid blob in the current pack file
|
||||
blobs: restic.Blobs{
|
||||
blobs: pack.Blobs{
|
||||
{
|
||||
Offset: 123,
|
||||
Length: 20000,
|
||||
@@ -368,7 +369,7 @@ func testStreamPack(t *testing.T, version uint) {
|
||||
|
||||
{
|
||||
// pass a blob that's too small
|
||||
blobs: restic.Blobs{
|
||||
blobs: pack.Blobs{
|
||||
{
|
||||
Offset: 123,
|
||||
Length: 10,
|
||||
@@ -523,7 +524,7 @@ func TestStreamPackFallback(t *testing.T) {
|
||||
|
||||
plaintext := rtest.Random(800, 42)
|
||||
blobID := restic.Hash(plaintext)
|
||||
blobs := restic.Blobs{
|
||||
blobs := pack.Blobs{
|
||||
{
|
||||
Length: uint(crypto.CiphertextLength(len(plaintext))),
|
||||
Offset: 0,
|
||||
|
||||
@@ -400,7 +400,7 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
|
||||
rtest.OK(t, err)
|
||||
|
||||
for pb := range idx.Values() {
|
||||
packID := pb.PackID
|
||||
packID := pb.PackID()
|
||||
if _, ok := packEntries[packID]; !ok {
|
||||
packEntries[packID] = make(map[restic.ID]struct{})
|
||||
}
|
||||
|
||||
@@ -1,46 +1,11 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
)
|
||||
|
||||
// Blob is one part of a file or a tree.
|
||||
type Blob struct {
|
||||
BlobHandle
|
||||
Length uint
|
||||
Offset uint
|
||||
UncompressedLength uint
|
||||
}
|
||||
|
||||
func (b Blob) String() string {
|
||||
return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v, uncompressed length %v>",
|
||||
b.Type, b.ID.Str(), b.Offset, b.Length, b.UncompressedLength)
|
||||
}
|
||||
|
||||
func (b Blob) DataLength() uint {
|
||||
if b.UncompressedLength != 0 {
|
||||
return b.UncompressedLength
|
||||
}
|
||||
return uint(crypto.PlaintextLength(int(b.Length)))
|
||||
}
|
||||
|
||||
func (b Blob) IsCompressed() bool {
|
||||
return b.UncompressedLength != 0
|
||||
}
|
||||
|
||||
type Blobs []Blob
|
||||
|
||||
func (b Blobs) Sort() {
|
||||
slices.SortFunc(b, func(a, b Blob) int {
|
||||
return cmp.Compare(a.Offset, b.Offset)
|
||||
})
|
||||
}
|
||||
|
||||
// PackBlob is one index entry for a blob in a pack file.
|
||||
// The interface intentionally omits the offset at which a blob is stored in the pack.
|
||||
// This ensures that pack file internals are not leaked.
|
||||
@@ -54,29 +19,6 @@ type PackBlob interface {
|
||||
IsCompressed() bool
|
||||
}
|
||||
|
||||
// PackedBlob is a blob stored within a file.
|
||||
type PackedBlob struct {
|
||||
Blob
|
||||
PackID ID
|
||||
}
|
||||
|
||||
type packBlob struct {
|
||||
PackedBlob
|
||||
}
|
||||
|
||||
func (pb packBlob) PackID() ID { return pb.PackedBlob.PackID }
|
||||
|
||||
func (pb packBlob) Handle() BlobHandle { return pb.BlobHandle }
|
||||
|
||||
func (pb packBlob) CiphertextLength() uint { return pb.Length }
|
||||
|
||||
func (pb packBlob) PlaintextLength() uint { return pb.DataLength() }
|
||||
|
||||
func (pb packBlob) IsCompressed() bool { return pb.Blob.IsCompressed() }
|
||||
|
||||
// AsPackBlob returns a PackBlob view of a PackedBlob.
|
||||
func AsPackBlob(pb PackedBlob) PackBlob { return packBlob{pb} }
|
||||
|
||||
// BlobHandle identifies a blob of a given type.
|
||||
type BlobHandle struct {
|
||||
ID ID
|
||||
|
||||
@@ -3,8 +3,6 @@ package restic
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
var blobTypeJSON = []struct {
|
||||
@@ -41,20 +39,3 @@ func TestBlobTypeJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobsSort(t *testing.T) {
|
||||
blobs := Blobs{
|
||||
{Offset: 100},
|
||||
{Offset: 0},
|
||||
{Offset: 50},
|
||||
}
|
||||
blobs.Sort()
|
||||
rtest.Equals(t, uint(0), blobs[0].Offset)
|
||||
rtest.Equals(t, uint(50), blobs[1].Offset)
|
||||
rtest.Equals(t, uint(100), blobs[2].Offset)
|
||||
}
|
||||
|
||||
func TestBlobsSortNilSlice(t *testing.T) {
|
||||
var blobs Blobs
|
||||
blobs.Sort()
|
||||
}
|
||||
|
||||
@@ -124,11 +124,6 @@ type SaverRemoverUnpacked[FT FileTypes] interface {
|
||||
RemoverUnpacked[FT]
|
||||
}
|
||||
|
||||
type PackBlobs struct {
|
||||
PackID ID
|
||||
Blobs Blobs
|
||||
}
|
||||
|
||||
type TerminalCounterFactory interface {
|
||||
// NewCounterTerminalOnly returns a new progress counter that is only shown if stdout points to a
|
||||
// terminal. It is not shown if --quiet or --json is specified.
|
||||
|
||||
Reference in New Issue
Block a user