mirror of
https://github.com/restic/restic.git
synced 2026-09-16 17:28:00 +00:00
restic: list pack header via ListPackHandles
Replace ListPack with ListPackHandles so callers only receive blob handles from pack headers, not layout fields.
This commit is contained in:
@@ -103,17 +103,17 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
|
|||||||
defer unlock()
|
defer unlock()
|
||||||
|
|
||||||
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
|
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
|
||||||
blobs, err := repo.ListPack(context.TODO(), id, size)
|
handles, err := repo.ListPackHandles(context.TODO(), id, size)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
|
rtest.Assert(t, len(handles) > 0, "a packfile should contain at least one blob")
|
||||||
|
|
||||||
switch blobs[0].Type {
|
switch handles[0].Type {
|
||||||
case restic.TreeBlob:
|
case restic.TreeBlob:
|
||||||
countTreePacks++
|
countTreePacks++
|
||||||
case restic.DataBlob:
|
case restic.DataBlob:
|
||||||
countDataPacks++
|
countDataPacks++
|
||||||
}
|
}
|
||||||
countBlobs += len(blobs)
|
countBlobs += len(handles)
|
||||||
return nil
|
return nil
|
||||||
}))
|
}))
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -473,18 +473,18 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
|||||||
delete(packIDs, idStr)
|
delete(packIDs, idStr)
|
||||||
}
|
}
|
||||||
debug.Log("Found pack %s", idStr)
|
debug.Log("Found pack %s", idStr)
|
||||||
blobs, err := f.repo.ListPack(ctx, id, size)
|
handles, err := f.repo.ListPackHandles(ctx, id, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, b := range blobs {
|
for _, h := range handles {
|
||||||
switch b.Type {
|
switch h.Type {
|
||||||
case restic.DataBlob:
|
case restic.DataBlob:
|
||||||
f.blobIDs[b.ID.String()] = struct{}{}
|
f.blobIDs[h.ID.String()] = struct{}{}
|
||||||
case restic.TreeBlob:
|
case restic.TreeBlob:
|
||||||
f.treeIDs[b.ID.String()] = struct{}{}
|
f.treeIDs[h.ID.String()] = struct{}{}
|
||||||
default:
|
default:
|
||||||
panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String()))
|
panic(fmt.Sprintf("unknown type %v in blob list", h.Type.String()))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Stop searching when all packs have been found
|
// Stop searching when all packs have been found
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func writePackDumpJSON(wr io.Writer, item any) error {
|
|||||||
func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
|
func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
|
||||||
var m sync.Mutex
|
var m sync.Mutex
|
||||||
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
|
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
|
||||||
blobs, err := repo.ListPack(ctx, id, size)
|
blobs, err := repo.listPack(ctx, id, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
printer.E("error for pack %v: %v", id.Str(), err)
|
printer.E("error for pack %v: %v", id.Str(), err)
|
||||||
return nil
|
return nil
|
||||||
@@ -135,7 +135,7 @@ func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts Exami
|
|||||||
printer.S(" ========================================")
|
printer.S(" ========================================")
|
||||||
printer.S(" inspect the pack itself")
|
printer.S(" inspect the pack itself")
|
||||||
|
|
||||||
blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
|
blobs, err := repo.listPack(ctx, id, int64(len(buf)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("pack %v: %v", id.Str(), err)
|
return fmt.Errorf("pack %v: %v", id.Str(), err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,13 +86,12 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
|
|||||||
blobs := restic.NewBlobSet()
|
blobs := restic.NewBlobSet()
|
||||||
|
|
||||||
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
|
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
|
||||||
entries, err := repo.ListPack(context.TODO(), id, size)
|
handles, err := repo.ListPackHandles(context.TODO(), id, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error listing pack %v: %v", id, err)
|
t.Fatalf("error listing pack %v: %v", id, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, entry := range entries {
|
for _, h := range handles {
|
||||||
h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
|
|
||||||
if blobs.Has(h) {
|
if blobs.Has(h) {
|
||||||
t.Errorf("ignoring duplicate blob %v", h)
|
t.Errorf("ignoring duplicate blob %v", h)
|
||||||
return nil
|
return nil
|
||||||
@@ -100,9 +99,9 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
|
|||||||
blobs.Insert(h)
|
blobs.Insert(h)
|
||||||
|
|
||||||
if random.Float32() <= p {
|
if random.Float32() <= p {
|
||||||
list1.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type})
|
list1.Insert(h)
|
||||||
} else {
|
} else {
|
||||||
list2.Insert(restic.BlobHandle{ID: entry.ID, Type: entry.Type})
|
list2.Insert(h)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
|
|||||||
|
|
||||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||||
if ids.Has(id) {
|
if ids.Has(id) {
|
||||||
blobs, err := repo.ListPack(ctx, id, size)
|
blobs, err := repo.listPack(ctx, id, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -783,7 +783,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
|
|||||||
// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
|
// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
|
||||||
worker := func() error {
|
worker := func() error {
|
||||||
for fi := range ch {
|
for fi := range ch {
|
||||||
entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
|
entries, err := r.listPack(wgCtx, fi.ID, fi.Size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
debug.Log("unable to list pack file %v", fi.ID.Str())
|
debug.Log("unable to list pack file %v", fi.ID.Str())
|
||||||
m.Lock()
|
m.Lock()
|
||||||
@@ -964,8 +964,8 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListPack returns the list of blobs saved in the pack id.
|
// listPack returns blob entries from the pack file header including offsets.
|
||||||
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
|
func (r *Repository) listPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
|
||||||
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
|
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
|
||||||
|
|
||||||
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
||||||
@@ -981,6 +981,19 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
|
|||||||
return entries, err
|
return entries, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListPackHandles returns the blob handles stored in the pack file header.
|
||||||
|
func (r *Repository) ListPackHandles(ctx context.Context, id restic.ID, size int64) ([]restic.BlobHandle, error) {
|
||||||
|
blobs, err := r.listPack(ctx, id, size)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
handles := make([]restic.BlobHandle, len(blobs))
|
||||||
|
for i, blob := range blobs {
|
||||||
|
handles[i] = blob.BlobHandle
|
||||||
|
}
|
||||||
|
return handles, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Delete calls backend.Delete() if implemented, and returns an error
|
// Delete calls backend.Delete() if implemented, and returns an error
|
||||||
// otherwise.
|
// otherwise.
|
||||||
func (r *Repository) Delete(ctx context.Context) error {
|
func (r *Repository) Delete(ctx context.Context) error {
|
||||||
|
|||||||
@@ -479,11 +479,11 @@ func TestListPack(t *testing.T) {
|
|||||||
return nil
|
return nil
|
||||||
}))
|
}))
|
||||||
|
|
||||||
blobs, err := repo.ListPack(context.TODO(), packID, size)
|
handles, err := repo.ListPackHandles(context.TODO(), packID, size)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
|
rtest.Assert(t, len(handles) == 1 && handles[0].ID == id, "unexpected blobs in pack: %v", handles)
|
||||||
|
|
||||||
rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as ListPack does not set IsMetadata in the backend.Handle")
|
rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as listPack does not set IsMetadata in the backend.Handle")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNoDoubleInit(t *testing.T) {
|
func TestNoDoubleInit(t *testing.T) {
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ type Repository interface {
|
|||||||
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
|
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
|
||||||
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
|
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
|
||||||
ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
|
ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
|
||||||
// ListPack returns the list of blobs saved in the pack id.
|
// ListPackHandles returns the blob handles stored in the pack file header.
|
||||||
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
|
ListPackHandles(ctx context.Context, id ID, packSize int64) ([]BlobHandle, error)
|
||||||
|
|
||||||
LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
|
LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
|
||||||
LoadBlobsFromPack(ctx context.Context, packID ID, blobs []BlobHandle, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error
|
LoadBlobsFromPack(ctx context.Context, packID ID, blobs []BlobHandle, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error
|
||||||
|
|||||||
Reference in New Issue
Block a user