mirror of
https://github.com/restic/restic.git
synced 2026-06-17 14:14:19 +00:00
restic: introduce Counter interface to decouple from ui/progress (#21861)
decouple restic and ui/progress packages
This commit is contained in:
@@ -553,12 +553,12 @@ func newJSONErrorPrinter(term ui.Terminal) *jsonErrorPrinter {
|
||||
}
|
||||
}
|
||||
|
||||
func (*jsonErrorPrinter) NewCounter(_ string) *progress.Counter {
|
||||
return nil
|
||||
func (*jsonErrorPrinter) NewCounter(_ string) restic.Counter {
|
||||
return restic.NoopCounter
|
||||
}
|
||||
|
||||
func (*jsonErrorPrinter) NewCounterTerminalOnly(_ string) *progress.Counter {
|
||||
return nil
|
||||
func (*jsonErrorPrinter) NewCounterTerminalOnly(_ string) restic.Counter {
|
||||
return restic.NoopCounter
|
||||
}
|
||||
|
||||
func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
|
||||
|
||||
@@ -273,7 +273,7 @@ func copyTree(ctx context.Context, srcRepo *repository.Repository, dstRepo resti
|
||||
}
|
||||
}
|
||||
|
||||
err := data.StreamTrees(ctx, srcRepo, restic.IDs{rootTreeID}, nil, func(treeID restic.ID) bool {
|
||||
err := data.StreamTrees(ctx, srcRepo, restic.IDs{rootTreeID}, restic.NoopCounter, func(treeID restic.ID) bool {
|
||||
handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}
|
||||
visited := visitedTrees.Has(handle)
|
||||
visitedTrees.Insert(handle)
|
||||
|
||||
@@ -235,7 +235,7 @@ func TestFindPackID(t *testing.T) {
|
||||
defer unlock()
|
||||
|
||||
// load Index
|
||||
rtest.OK(t, repo.LoadIndex(ctx, nil))
|
||||
rtest.OK(t, repo.LoadIndex(ctx, restic.NoopTerminalCounterFactory))
|
||||
// go through all index entries and collect data and tree packfile(s)
|
||||
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackBlob) {
|
||||
switch blob.Handle().Type {
|
||||
|
||||
@@ -67,7 +67,7 @@ func testListBlobs(t testing.TB, gopts global.Options) (blobSetFromIndex restic.
|
||||
defer unlock()
|
||||
|
||||
// make sure the index is loaded
|
||||
rtest.OK(t, repo.LoadIndex(ctx, nil))
|
||||
rtest.OK(t, repo.LoadIndex(ctx, restic.NoopTerminalCounterFactory))
|
||||
|
||||
// get blobs from index
|
||||
blobSetFromIndex = restic.NewIDSet()
|
||||
|
||||
@@ -214,8 +214,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
|
||||
return nil
|
||||
}
|
||||
|
||||
func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer, progress *statsui.Progress) error {
|
||||
progress.ProcessSnapshot()
|
||||
func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer, sp *statsui.Progress) error {
|
||||
sp.ProcessSnapshot()
|
||||
if snapshot.Tree == nil {
|
||||
return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
|
||||
}
|
||||
@@ -225,12 +225,12 @@ func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic
|
||||
if opts.countMode == countModeRawData {
|
||||
// count just the sizes of unique blobs; we don't need to walk the tree
|
||||
// ourselves in this case, since a nifty function does it for us
|
||||
return data.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
|
||||
return data.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, restic.NoopCounter)
|
||||
}
|
||||
|
||||
hardLinkIndex := restorer.NewHardlinkIndex[struct{}]()
|
||||
err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{
|
||||
ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex, progress),
|
||||
ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex, sp),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
|
||||
|
||||
@@ -267,7 +267,7 @@ func listTreePacks(gopts global.Options, t *testing.T) restic.IDSet {
|
||||
rtest.OK(t, err)
|
||||
defer unlock()
|
||||
|
||||
rtest.OK(t, r.LoadIndex(ctx, nil))
|
||||
rtest.OK(t, r.LoadIndex(ctx, restic.NoopTerminalCounterFactory))
|
||||
treePacks = restic.NewIDSet()
|
||||
return r.ListBlobs(ctx, func(pb restic.PackBlob) {
|
||||
if pb.Handle().Type == restic.TreeBlob {
|
||||
@@ -315,7 +315,7 @@ func removePacksExcept(gopts global.Options, t testing.TB, keep restic.IDSet, re
|
||||
defer unlock()
|
||||
|
||||
// Get all tree packs
|
||||
rtest.OK(t, r.LoadIndex(ctx, nil))
|
||||
rtest.OK(t, r.LoadIndex(ctx, restic.NoopTerminalCounterFactory))
|
||||
|
||||
treePacks := restic.NewIDSet()
|
||||
rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackBlob) {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
// Checker runs various checks on a repository. It is advisable to create an
|
||||
@@ -139,7 +138,7 @@ func (c *Checker) loadActiveTrees(ctx context.Context, snapshotFilter *data.Snap
|
||||
// Structure checks that for all snapshots all referenced data blobs and
|
||||
// subtrees are available in the index. errChan is closed after all trees have
|
||||
// been traversed.
|
||||
func (c *Checker) Structure(ctx context.Context, p *progress.Counter, errChan chan<- error) {
|
||||
func (c *Checker) Structure(ctx context.Context, p restic.Counter, errChan chan<- error) {
|
||||
trees, errs := c.loadActiveTrees(ctx, c.snapshotFilter, c.args)
|
||||
p.SetMax(uint64(len(trees)))
|
||||
debug.Log("need to check %d trees from snapshots, %d errs returned", len(trees), len(errs))
|
||||
@@ -293,7 +292,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
|
||||
// with an unmodified parameter list
|
||||
// Otherwise it calculates the packfiles needed, gets their sizes from the full
|
||||
// packfile set and submits them to repository.ReadPacks()
|
||||
func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p *progress.Counter, errChan chan<- error) {
|
||||
func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p restic.Counter, errChan chan<- error) {
|
||||
// no snapshot filtering, pass through
|
||||
if !c.IsFiltered() {
|
||||
c.Checker.ReadPacks(ctx, filter, p, errChan)
|
||||
|
||||
@@ -49,7 +49,7 @@ func checkStruct(chkr *checker.Checker) []error {
|
||||
return []error{err}
|
||||
}
|
||||
return collectErrors(context.TODO(), func(ctx context.Context, errChan chan<- error) {
|
||||
chkr.Structure(ctx, nil, errChan)
|
||||
chkr.Structure(ctx, restic.NoopCounter, errChan)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ func checkData(chkr *checker.Checker) []error {
|
||||
func(ctx context.Context, errCh chan<- error) {
|
||||
chkr.ReadPacks(ctx, func(packs map[restic.ID]int64) map[restic.ID]int64 {
|
||||
return packs
|
||||
}, nil, errCh)
|
||||
}, restic.NoopCounter, errCh)
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func TestCheckRepo(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func TestMissingPack(t *testing.T) {
|
||||
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: packID.String()}))
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
@@ -126,7 +126,7 @@ func TestUnreferencedPack(t *testing.T) {
|
||||
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: indexID.String()}))
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
@@ -163,7 +163,7 @@ func TestUnreferencedBlobs(t *testing.T) {
|
||||
sort.Sort(unusedBlobsBySnapshot)
|
||||
|
||||
chkr := checker.New(repo, true)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
@@ -205,7 +205,7 @@ func TestModifiedIndex(t *testing.T) {
|
||||
test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher())))
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) == 0 {
|
||||
t.Fatalf("expected errors not found")
|
||||
}
|
||||
@@ -224,7 +224,7 @@ func TestDuplicatePacksInIndex(t *testing.T) {
|
||||
defer cleanup()
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(hints) == 0 {
|
||||
t.Fatalf("did not get expected checker hints for duplicate packs in indexes")
|
||||
}
|
||||
@@ -364,7 +364,7 @@ func TestCheckerModifiedData(t *testing.T) {
|
||||
|
||||
chkr := checker.New(checkRepo, false)
|
||||
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
@@ -428,7 +428,7 @@ func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
|
||||
}
|
||||
|
||||
chkr := checker.New(checkRepo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
@@ -545,7 +545,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
|
||||
delayRepo.Unblock()
|
||||
}()
|
||||
|
||||
hints, errs := chkr.LoadIndex(ctx, nil)
|
||||
hints, errs := chkr.LoadIndex(ctx, restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
@@ -573,7 +573,7 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun
|
||||
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) > 0 {
|
||||
defer cleanup()
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
func TestCheckRepo(t testing.TB, repo checkerRepository) {
|
||||
chkr := New(repo, true)
|
||||
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("errors loading index: %v", errs)
|
||||
}
|
||||
@@ -36,7 +36,7 @@ func TestCheckRepo(t testing.TB, repo checkerRepository) {
|
||||
|
||||
// structure
|
||||
errChan = make(chan error)
|
||||
go chkr.Structure(context.TODO(), nil, errChan)
|
||||
go chkr.Structure(context.TODO(), restic.NoopCounter, errChan)
|
||||
|
||||
for err := range errChan {
|
||||
t.Error(err)
|
||||
@@ -55,7 +55,7 @@ func TestCheckRepo(t testing.TB, repo checkerRepository) {
|
||||
errChan = make(chan error)
|
||||
go chkr.ReadPacks(context.TODO(), func(packs map[restic.ID]int64) map[restic.ID]int64 {
|
||||
return packs
|
||||
}, nil, errChan)
|
||||
}, restic.NoopCounter, errChan)
|
||||
|
||||
for err := range errChan {
|
||||
t.Error(err)
|
||||
|
||||
@@ -5,12 +5,11 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
// FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data
|
||||
// blobs) to the set blobs. Already seen tree blobs will not be visited again.
|
||||
func FindUsedBlobs(ctx context.Context, repo restic.Loader, treeIDs restic.IDs, blobs restic.FindBlobSet, p *progress.Counter) error {
|
||||
func FindUsedBlobs(ctx context.Context, repo restic.Loader, treeIDs restic.IDs, blobs restic.FindBlobSet, p restic.Counter) error {
|
||||
var lock sync.Mutex
|
||||
|
||||
return StreamTrees(ctx, repo, treeIDs, p, func(treeID restic.ID) bool {
|
||||
|
||||
@@ -182,12 +182,12 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
|
||||
t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str())
|
||||
|
||||
usedBlobs := restic.NewBlobSet()
|
||||
err := data.FindUsedBlobs(context.TODO(), repo, restic.IDs{*snapshot.Tree}, usedBlobs, nil)
|
||||
err := data.FindUsedBlobs(context.TODO(), repo, restic.IDs{*snapshot.Tree}, usedBlobs, restic.NoopCounter)
|
||||
if err != nil {
|
||||
t.Fatalf("FindUsedBlobs returned error: %v", err)
|
||||
}
|
||||
|
||||
err = data.FindUsedBlobs(context.TODO(), ForbiddenRepo{}, restic.IDs{*snapshot.Tree}, usedBlobs, nil)
|
||||
err = data.FindUsedBlobs(context.TODO(), ForbiddenRepo{}, restic.IDs{*snapshot.Tree}, usedBlobs, restic.NoopCounter)
|
||||
if err != nil {
|
||||
t.Fatalf("FindUsedBlobs returned error: %v", err)
|
||||
}
|
||||
@@ -202,7 +202,7 @@ func BenchmarkFindUsedBlobs(b *testing.B) {
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
blobs := restic.NewBlobSet()
|
||||
err := data.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, blobs, nil)
|
||||
err := data.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, blobs, restic.NoopCounter)
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -93,7 +92,7 @@ func loadTreeWorker(
|
||||
|
||||
// filterTree receives the result of a tree load and queues new trees for loading and processing.
|
||||
func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, loaderChan chan<- trackedID, hugeTreeLoaderChan chan<- trackedID,
|
||||
in <-chan trackedTreeItem, skip func(tree restic.ID) bool, p *progress.Counter) {
|
||||
in <-chan trackedTreeItem, skip func(tree restic.ID) bool, p restic.Counter) {
|
||||
|
||||
var (
|
||||
inCh = in
|
||||
@@ -119,7 +118,7 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load
|
||||
|
||||
if skip(nextTreeID.ID) {
|
||||
rootCounter[nextTreeID.rootIdx]--
|
||||
if p != nil && rootCounter[nextTreeID.rootIdx] == 0 {
|
||||
if rootCounter[nextTreeID.rootIdx] == 0 {
|
||||
p.Add(1)
|
||||
}
|
||||
continue
|
||||
@@ -172,7 +171,7 @@ func filterTrees(ctx context.Context, repo restic.Loader, trees restic.IDs, load
|
||||
rootCounter[j.rootIdx]++
|
||||
}
|
||||
// the progress check must happen after j.Subtrees was added to the backlog
|
||||
if p != nil && rootCounter[j.rootIdx] == 0 {
|
||||
if rootCounter[j.rootIdx] == 0 {
|
||||
p.Add(1)
|
||||
}
|
||||
}
|
||||
@@ -188,7 +187,7 @@ func StreamTrees(
|
||||
ctx context.Context,
|
||||
repo restic.Loader,
|
||||
trees restic.IDs,
|
||||
p *progress.Counter,
|
||||
p restic.Counter,
|
||||
skip func(tree restic.ID) bool,
|
||||
process func(id restic.ID, error error, nodes TreeNodeIterator) error,
|
||||
) error {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"github.com/restic/restic/internal/data"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
type MetaDirData struct {
|
||||
@@ -326,7 +327,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = d.root.repo.LoadIndex(ctx, nil)
|
||||
err = d.root.repo.LoadIndex(ctx, restic.NoopTerminalCounterFactory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -244,7 +243,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
|
||||
}
|
||||
|
||||
// ReadPacks loads data from specified packs and checks the integrity.
|
||||
func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p *progress.Counter, errChan chan<- error) {
|
||||
func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p restic.Counter, errChan chan<- error) {
|
||||
defer close(errChan)
|
||||
|
||||
// compute pack size using index entries
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestGapInBlobs(t *testing.T) {
|
||||
repo, _, cleanup := TestFromFixture(t, checkerTestData)
|
||||
defer cleanup()
|
||||
|
||||
err := repo.LoadIndex(context.TODO(), nil)
|
||||
err := repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
rtest.OK(t, err)
|
||||
|
||||
repoPacks, err := pack.Size(context.TODO(), repo, false)
|
||||
@@ -90,7 +90,7 @@ func runReadPacks(chkr *Checker) []error {
|
||||
func(ctx context.Context, errCh chan<- error) {
|
||||
chkr.ReadPacks(ctx, func(packs map[restic.ID]int64) map[restic.ID]int64 {
|
||||
return packs
|
||||
}, nil, errCh)
|
||||
}, restic.NoopCounter, errCh)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ func setupChecker(t *testing.T, wrap func(backend.Backend) backend.Backend) *Che
|
||||
chkr := newChecker(checkRepo)
|
||||
|
||||
// make sure the index is loaded
|
||||
err := checkRepo.LoadIndex(context.TODO(), nil)
|
||||
err := checkRepo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
rtest.OK(t, err)
|
||||
|
||||
return chkr
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -281,7 +280,7 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
|
||||
func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p restic.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
|
||||
indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -290,31 +289,27 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if p != nil {
|
||||
var numIndexFiles uint64
|
||||
err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
|
||||
if loadedIDs.Has(id) {
|
||||
// skip already loaded indexes
|
||||
return nil
|
||||
}
|
||||
numIndexFiles++
|
||||
var numIndexFiles uint64
|
||||
err = indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
|
||||
if loadedIDs.Has(id) {
|
||||
// skip already loaded indexes
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.SetMax(numIndexFiles)
|
||||
defer p.Done()
|
||||
numIndexFiles++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.SetMax(numIndexFiles)
|
||||
defer p.Done()
|
||||
|
||||
err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error {
|
||||
if loadedIDs.Has(id) {
|
||||
// skip already loaded indexes
|
||||
return nil
|
||||
}
|
||||
if p != nil {
|
||||
p.Add(1)
|
||||
}
|
||||
p.Add(1)
|
||||
if cb != nil {
|
||||
err = cb(id, idx, err)
|
||||
}
|
||||
@@ -368,8 +363,8 @@ func (mi *MasterIndex) prepareIncrementalLoad(ctx context.Context, indexList res
|
||||
}
|
||||
|
||||
type MasterIndexRewriteOpts struct {
|
||||
SaveProgress *progress.Counter
|
||||
DeleteProgress func() *progress.Counter
|
||||
SaveProgress restic.Counter
|
||||
DeleteProgress func() restic.Counter
|
||||
DeleteReport func(id restic.ID, err error)
|
||||
}
|
||||
|
||||
@@ -396,6 +391,9 @@ func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked[restic.
|
||||
}
|
||||
|
||||
p := opts.SaveProgress
|
||||
if p == nil {
|
||||
p = restic.NoopCounter
|
||||
}
|
||||
p.SetMax(uint64(len(indexes)))
|
||||
|
||||
// reset state which is not necessary for Rewrite and just consumes a lot of memory
|
||||
@@ -553,7 +551,7 @@ func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked[restic.
|
||||
return fmt.Errorf("failed to rewrite indexes: %w", err)
|
||||
}
|
||||
|
||||
p = nil
|
||||
p = restic.NoopCounter
|
||||
if opts.DeleteProgress != nil {
|
||||
p = opts.DeleteProgress()
|
||||
}
|
||||
@@ -571,7 +569,7 @@ func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked[restic.
|
||||
// It is only intended for use by prune with the UnsafeRecovery option.
|
||||
//
|
||||
// Must not be called concurrently to any other MasterIndex operation.
|
||||
func (mi *MasterIndex) SaveFallback(ctx context.Context, repo restic.SaverRemoverUnpacked[restic.FileType], excludePacks restic.IDSet, p *progress.Counter) error {
|
||||
func (mi *MasterIndex) SaveFallback(ctx context.Context, repo restic.SaverRemoverUnpacked[restic.FileType], excludePacks restic.IDSet, p restic.Counter) error {
|
||||
p.SetMax(uint64(len(mi.Packs(excludePacks))))
|
||||
|
||||
mi.idxMutex.Lock()
|
||||
|
||||
@@ -438,18 +438,18 @@ func testIndexSave(t *testing.T, version uint) {
|
||||
return idx.Rewrite(context.TODO(), repo, nil, restic.NewIDSet(), nil, index.MasterIndexRewriteOpts{})
|
||||
}},
|
||||
{"SaveFallback", func(idx *index.MasterIndex, repo restic.Unpacked[restic.FileType]) error {
|
||||
err := restic.ParallelRemove(context.TODO(), repo, idx.IDs(), restic.IndexFile, nil, nil)
|
||||
err := restic.ParallelRemove(context.TODO(), repo, idx.IDs(), restic.IndexFile, nil, restic.NoopCounter)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return idx.SaveFallback(context.TODO(), repo, restic.NewIDSet(), nil)
|
||||
return idx.SaveFallback(context.TODO(), repo, restic.NewIDSet(), restic.NoopCounter)
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
repo, unpacked := createFilledRepo(t, 3, version)
|
||||
|
||||
idx := index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
|
||||
blobs := make(map[pack.PackedBlob]struct{})
|
||||
for pb := range idx.Values() {
|
||||
blobs[*pb] = struct{}{}
|
||||
@@ -457,7 +457,7 @@ func testIndexSave(t *testing.T, version uint) {
|
||||
|
||||
rtest.OK(t, test.saver(idx, unpacked))
|
||||
idx = index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
|
||||
|
||||
for pb := range idx.Values() {
|
||||
if _, ok := blobs[*pb]; ok {
|
||||
@@ -482,7 +482,7 @@ func testIndexSavePartial(t *testing.T, version uint) {
|
||||
|
||||
// capture blob list before adding fourth snapshot
|
||||
idx := index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
|
||||
blobs := make(map[pack.PackedBlob]struct{})
|
||||
for pb := range idx.Values() {
|
||||
blobs[*pb] = struct{}{}
|
||||
@@ -497,12 +497,12 @@ func testIndexSavePartial(t *testing.T, version uint) {
|
||||
|
||||
// rewrite index and remove pack files of new snapshot
|
||||
idx = index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
|
||||
rtest.OK(t, idx.Rewrite(context.TODO(), unpacked, newPacks, nil, nil, index.MasterIndexRewriteOpts{}))
|
||||
|
||||
// check blobs
|
||||
idx = index.NewMasterIndex()
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
|
||||
rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
|
||||
for pb := range idx.Values() {
|
||||
if _, ok := blobs[*pb]; ok {
|
||||
delete(blobs, *pb)
|
||||
@@ -513,7 +513,7 @@ func testIndexSavePartial(t *testing.T, version uint) {
|
||||
rtest.Equals(t, 0, len(blobs), "saved index is missing blobs")
|
||||
|
||||
// remove pack files to make check happy
|
||||
rtest.OK(t, restic.ParallelRemove(context.TODO(), unpacked, newPacks, restic.PackFile, nil, nil))
|
||||
rtest.OK(t, restic.ParallelRemove(context.TODO(), unpacked, newPacks, restic.PackFile, nil, restic.NoopCounter))
|
||||
|
||||
checker.TestCheckRepo(t, repo)
|
||||
}
|
||||
@@ -620,14 +620,14 @@ func TestRewriteOversizedIndex(t *testing.T) {
|
||||
|
||||
// construct master index for the oversized index
|
||||
mi := index.NewMasterIndex()
|
||||
rtest.OK(t, mi.Load(context.Background(), repo, nil, nil))
|
||||
rtest.OK(t, mi.Load(context.Background(), repo, restic.NoopCounter, nil))
|
||||
|
||||
// rewrite the index
|
||||
rtest.OK(t, mi.Rewrite(context.Background(), unpacked, nil, nil, nil, index.MasterIndexRewriteOpts{}))
|
||||
|
||||
// load the rewritten indexes
|
||||
mi2 := index.NewMasterIndex()
|
||||
rtest.OK(t, mi2.Load(context.Background(), repo, nil, nil))
|
||||
rtest.OK(t, mi2.Load(context.Background(), repo, restic.NoopCounter, nil))
|
||||
|
||||
// verify that blobs are still in the index
|
||||
for _, blob := range blobs {
|
||||
@@ -685,7 +685,7 @@ func TestRewriteSplitPacks(t *testing.T) {
|
||||
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID()), nil, nil, index.MasterIndexRewriteOpts{}))
|
||||
|
||||
mi = index.NewMasterIndex()
|
||||
rtest.OK(t, mi.Load(context.TODO(), repo, nil, nil))
|
||||
rtest.OK(t, mi.Load(context.TODO(), repo, restic.NoopCounter, nil))
|
||||
|
||||
// test that all blobs are still in the index
|
||||
for _, blob := range []*pack.PackedBlob{blob1, blob2} {
|
||||
@@ -746,7 +746,7 @@ func TestRewriteFullPacks(t *testing.T) {
|
||||
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
|
||||
|
||||
mi2 := index.NewMasterIndex()
|
||||
rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
|
||||
rtest.OK(t, mi2.Load(context.TODO(), repo, restic.NoopCounter, nil))
|
||||
|
||||
afterRewrite := mi2.IDs()
|
||||
rtest.Equals(t, 2, len(afterRewrite))
|
||||
|
||||
@@ -23,7 +23,7 @@ func TestAllIndexBlobs(t *testing.T) {
|
||||
return nil
|
||||
}))
|
||||
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
|
||||
|
||||
fromMaster := restic.NewBlobSet()
|
||||
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {
|
||||
|
||||
@@ -153,7 +153,7 @@ func TestPruneSmall(t *testing.T) {
|
||||
|
||||
// and reopen repository with default packsize
|
||||
repo = repository.TestOpenBackend(t, be)
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
|
||||
|
||||
opts := repository.PruneOptions{
|
||||
MaxRepackBytes: math.MaxUint64,
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -37,7 +36,7 @@ func CopyBlobs(
|
||||
dstUploader restic.BlobSaverWithAsync,
|
||||
packs restic.IDSet,
|
||||
keepBlobs repackBlobSet,
|
||||
p *progress.Counter,
|
||||
p restic.Counter,
|
||||
logf LogFunc,
|
||||
) error {
|
||||
debug.Log("repacking %d packs while keeping %d blobs", len(packs), keepBlobs.Len())
|
||||
@@ -62,7 +61,7 @@ func repack(
|
||||
uploader restic.BlobSaverWithAsync,
|
||||
packs restic.IDSet,
|
||||
keepBlobs repackBlobSet,
|
||||
p *progress.Counter,
|
||||
p restic.Counter,
|
||||
logf LogFunc,
|
||||
) error {
|
||||
|
||||
|
||||
@@ -150,7 +150,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
|
||||
|
||||
func repack(t *testing.T, repo *repository.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) {
|
||||
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, nil, nil)
|
||||
return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, restic.NoopCounter, nil)
|
||||
}))
|
||||
|
||||
for id := range packs {
|
||||
@@ -163,7 +163,7 @@ func rebuildAndReloadIndex(t *testing.T, repo *repository.Repository) {
|
||||
ReadAllPacks: true,
|
||||
}, progress.NewNoopPrinter()))
|
||||
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
|
||||
}
|
||||
|
||||
func TestRepack(t *testing.T) {
|
||||
@@ -272,7 +272,7 @@ func testRepackCopy(t *testing.T, version uint) {
|
||||
copyPacks := findPacksForBlobs(t, repo, keepBlobs)
|
||||
|
||||
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
return repository.CopyBlobs(ctx, repo, dstRepo, uploader, copyPacks, keepBlobs, nil, nil)
|
||||
return repository.CopyBlobs(ctx, repo, dstRepo, uploader, copyPacks, keepBlobs, restic.NoopCounter, nil)
|
||||
}))
|
||||
rebuildAndReloadIndex(t, dstRepo)
|
||||
|
||||
@@ -310,7 +310,7 @@ func testRepackWrongBlob(t *testing.T, version uint) {
|
||||
rewritePacks := findPacksForBlobs(t, repo, keepBlobs)
|
||||
|
||||
err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, nil, nil)
|
||||
return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, restic.NoopCounter, nil)
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected repack to fail but got no error")
|
||||
@@ -359,7 +359,7 @@ func testRepackBlobFallback(t *testing.T, version uint) {
|
||||
|
||||
// repack must fallback to valid copy
|
||||
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, nil, nil)
|
||||
return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, restic.NoopCounter, nil)
|
||||
}))
|
||||
|
||||
keepBlobs = restic.NewBlobSet(restic.BlobHandle{Type: restic.DataBlob, ID: id})
|
||||
|
||||
@@ -32,7 +32,7 @@ func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions,
|
||||
|
||||
} else {
|
||||
printer.P("loading indexes...\n")
|
||||
err := repo.loadIndexWithCallback(ctx, nil, func(id restic.ID, _ *index.Index, err error) error {
|
||||
err := repo.loadIndexWithCallback(ctx, restic.NoopTerminalCounterFactory, func(id restic.ID, _ *index.Index, err error) error {
|
||||
if err != nil {
|
||||
printer.E("removing invalid index %v: %v\n", id, err)
|
||||
obsoleteIndexes = append(obsoleteIndexes, id)
|
||||
@@ -109,7 +109,7 @@ func rewriteIndexFiles(ctx context.Context, repo *Repository, removePacks restic
|
||||
bar := printer.NewCounter("indexes processed")
|
||||
return repo.idx.Rewrite(ctx, &internalRepository{repo}, removePacks, oldIndexes, extraObsolete, index.MasterIndexRewriteOpts{
|
||||
SaveProgress: bar,
|
||||
DeleteProgress: func() *progress.Counter {
|
||||
DeleteProgress: func() restic.Counter {
|
||||
return printer.NewCounter("old indexes deleted")
|
||||
},
|
||||
DeleteReport: func(id restic.ID, err error) {
|
||||
|
||||
@@ -131,7 +131,7 @@ func testRepairBrokenPack(t *testing.T, version uint) {
|
||||
|
||||
rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, progress.NewNoopPrinter()))
|
||||
// reload index
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
|
||||
|
||||
packsAfter := listPacks(t, repo)
|
||||
blobsAfter := listBlobs(repo)
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
@@ -716,10 +715,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p restic.TerminalCounterFact
|
||||
func (r *Repository) loadIndexWithCallback(ctx context.Context, p restic.TerminalCounterFactory, cb func(id restic.ID, idx *index.Index, err error) error) error {
|
||||
debug.Log("Loading index")
|
||||
|
||||
var bar *progress.Counter
|
||||
if p != nil {
|
||||
bar = p.NewCounterTerminalOnly("index files loaded")
|
||||
}
|
||||
bar := p.NewCounterTerminalOnly("index files loaded")
|
||||
|
||||
err := r.idx.Load(ctx, r, bar, cb)
|
||||
if err != nil {
|
||||
@@ -758,7 +754,7 @@ func (r *Repository) loadIndexWithCallback(ctx context.Context, p restic.Termina
|
||||
// createIndexFromPacks creates a new index by reading all given pack files (with sizes).
|
||||
// The index is added to the MasterIndex but not marked as finalized.
|
||||
// Returned is the list of pack files which could not be read.
|
||||
func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[restic.ID]int64, p *progress.Counter) (invalid restic.IDs, err error) {
|
||||
func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[restic.ID]int64, p restic.Counter) (invalid restic.IDs, err error) {
|
||||
var m sync.Mutex
|
||||
|
||||
debug.Log("Loading index from pack files")
|
||||
|
||||
@@ -322,7 +322,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
|
||||
repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
|
||||
defer cleanup()
|
||||
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
|
||||
}
|
||||
|
||||
// loadIndex loads the index id from backend and returns it.
|
||||
@@ -380,7 +380,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
|
||||
rtest.OK(t, err)
|
||||
repo := repository.TestOpenBackend(t, &damageOnceBackend{Backend: be})
|
||||
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
|
||||
}
|
||||
|
||||
// saveRandomDataBlobs generates random data blobs and saves them to the repository.
|
||||
|
||||
@@ -167,7 +167,7 @@ func TestNewLock(_ *testing.T, repo *Repository, exclusive bool) (*restic.Lock,
|
||||
func TestCheckRepo(t testing.TB, repo *Repository) {
|
||||
chkr := newChecker(repo)
|
||||
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
|
||||
if len(errs) != 0 {
|
||||
t.Fatalf("errors loading index: %v", errs)
|
||||
}
|
||||
@@ -188,7 +188,7 @@ func TestCheckRepo(t testing.TB, repo *Repository) {
|
||||
errChan = make(chan error)
|
||||
go chkr.ReadPacks(context.TODO(), func(packs map[restic.ID]int64) map[restic.ID]int64 {
|
||||
return packs
|
||||
}, nil, errChan)
|
||||
}, restic.NoopCounter, errChan)
|
||||
|
||||
for err := range errChan {
|
||||
t.Error(err)
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -54,7 +53,7 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f
|
||||
|
||||
// ParallelRemove deletes the given fileList of fileType in parallel.
|
||||
// If report returns an error, it aborts.
|
||||
func ParallelRemove[FT FileTypes](ctx context.Context, repo RemoverUnpacked[FT], fileList IDSet, fileType FT, report func(id ID, err error) error, bar *progress.Counter) error {
|
||||
func ParallelRemove[FT FileTypes](ctx context.Context, repo RemoverUnpacked[FT], fileList IDSet, fileType FT, report func(id ID, err error) error, bar Counter) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
wg.SetLimit(int(repo.Connections())) // deleting files is IO-bound
|
||||
|
||||
|
||||
@@ -3,11 +3,10 @@ package restic
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
type mockRemoverUnpacked struct {
|
||||
@@ -26,6 +25,15 @@ func NewTestID(i byte) ID {
|
||||
return Hash([]byte{i})
|
||||
}
|
||||
|
||||
type testCounter struct {
|
||||
value atomic.Uint64
|
||||
}
|
||||
|
||||
func (c *testCounter) Add(v uint64) { c.value.Add(v) }
|
||||
func (c *testCounter) SetMax(_ uint64) {}
|
||||
func (c *testCounter) Get() (uint64, uint64) { return c.value.Load(), 0 }
|
||||
func (c *testCounter) Done() {}
|
||||
|
||||
func TestParallelRemove(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -83,7 +91,7 @@ func TestParallelRemove(t *testing.T) {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
repo := &mockRemoverUnpacked{removeUnpacked: test.removeUnpacked}
|
||||
reportIDSet := NewIDSet()
|
||||
bar := progress.NewCounter(time.Millisecond, 0, func(value uint64, total uint64, runtime time.Duration, final bool) {})
|
||||
bar := &testCounter{}
|
||||
report := func(id ID, err error) error {
|
||||
if err == nil {
|
||||
mu.Lock()
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package restic
|
||||
|
||||
// Counter tracks progress for long-running operations.
|
||||
type Counter interface {
|
||||
Add(uint64)
|
||||
SetMax(uint64)
|
||||
Get() (uint64, uint64)
|
||||
Done()
|
||||
}
|
||||
|
||||
type noopCounter struct{}
|
||||
|
||||
func (noopCounter) Add(uint64) {}
|
||||
func (noopCounter) SetMax(uint64) {}
|
||||
func (noopCounter) Get() (uint64, uint64) { return 0, 0 }
|
||||
func (noopCounter) Done() {}
|
||||
|
||||
// NoopCounter is a Counter that discards all updates.
|
||||
var NoopCounter Counter = noopCounter{}
|
||||
|
||||
type noopTerminalCounterFactory struct{}
|
||||
|
||||
func (noopTerminalCounterFactory) NewCounterTerminalOnly(string) Counter {
|
||||
return NoopCounter
|
||||
}
|
||||
|
||||
// NoopTerminalCounterFactory is a TerminalCounterFactory that returns NoopCounter.
|
||||
var NoopTerminalCounterFactory TerminalCounterFactory = noopTerminalCounterFactory{}
|
||||
@@ -0,0 +1,18 @@
|
||||
package restic_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
func TestNoopCounter(_ *testing.T) {
|
||||
c := restic.NoopCounter
|
||||
c.Add(1)
|
||||
c.SetMax(42)
|
||||
c.Done()
|
||||
v, max := c.Get()
|
||||
if v != 0 || max != 0 {
|
||||
panic("noop counter must not change")
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
// ErrInvalidData is used to report that a file is corrupted
|
||||
@@ -125,7 +124,7 @@ type SaverRemoverUnpacked[FT FileTypes] interface {
|
||||
type TerminalCounterFactory interface {
|
||||
// NewCounterTerminalOnly returns a new progress counter that is only shown if stdout points to a
|
||||
// terminal. It is not shown if --quiet or --json is specified.
|
||||
NewCounterTerminalOnly(description string) *progress.Counter
|
||||
NewCounterTerminalOnly(description string) Counter
|
||||
}
|
||||
|
||||
// Lister allows listing files in a backend.
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
restoreui "github.com/restic/restic/internal/ui/restore"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
@@ -621,7 +620,7 @@ const nVerifyWorkers = 8
|
||||
// have been successfully written to dst. It stops when it encounters an
|
||||
// error. It returns that error and the number of files it has successfully
|
||||
// verified.
|
||||
func (res *Restorer) VerifyFiles(ctx context.Context, dst string, countRestoredFiles uint64, p *progress.Counter) (int, error) {
|
||||
func (res *Restorer) VerifyFiles(ctx context.Context, dst string, countRestoredFiles uint64, p restic.Counter) (int, error) {
|
||||
type mustCheck struct {
|
||||
node *data.Node
|
||||
path string
|
||||
|
||||
@@ -400,7 +400,7 @@ func TestRestorer(t *testing.T) {
|
||||
}
|
||||
|
||||
if len(test.ErrorsMust)+len(test.ErrorsMay) == 0 {
|
||||
_, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil)
|
||||
_, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, restic.NoopCounter)
|
||||
rtest.OK(t, err)
|
||||
}
|
||||
|
||||
@@ -886,7 +886,7 @@ func TestVerifyCancel(t *testing.T) {
|
||||
return err
|
||||
}
|
||||
|
||||
nverified, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil)
|
||||
nverified, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, restic.NoopCounter)
|
||||
rtest.Equals(t, 0, nverified)
|
||||
rtest.Assert(t, err != nil, "nil error from VerifyFiles")
|
||||
rtest.Equals(t, 1, len(errs))
|
||||
@@ -964,7 +964,7 @@ func saveSnapshotsAndOverwrite(t *testing.T, baseSnapshot Snapshot, overwriteSna
|
||||
countRestoredFiles, err := res.RestoreTo(ctx, tempdir)
|
||||
rtest.OK(t, err)
|
||||
|
||||
_, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil)
|
||||
_, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, restic.NoopCounter)
|
||||
rtest.OK(t, err)
|
||||
|
||||
return tempdir
|
||||
@@ -1248,7 +1248,7 @@ func TestRestoreModified(t *testing.T) {
|
||||
res := NewRestorer(repo, sn, Options{Overwrite: OverwriteIfChanged})
|
||||
countRestoredFiles, err := res.RestoreTo(ctx, tempdir)
|
||||
rtest.OK(t, err)
|
||||
n, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil)
|
||||
n, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, restic.NoopCounter)
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, 2, n, "unexpected number of verified files")
|
||||
}
|
||||
@@ -1550,6 +1550,6 @@ func TestRestorerLongPath(t *testing.T) {
|
||||
|
||||
countRestoredFiles, err := res.RestoreTo(ctx, tmp)
|
||||
rtest.OK(t, err)
|
||||
_, err = res.VerifyFiles(ctx, tmp, countRestoredFiles, nil)
|
||||
_, err = res.VerifyFiles(ctx, tmp, countRestoredFiles, restic.NoopCounter)
|
||||
rtest.OK(t, err)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package progress
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// A Func is a callback for a Counter.
|
||||
@@ -20,6 +22,8 @@ type Counter struct {
|
||||
value, max atomic.Uint64
|
||||
}
|
||||
|
||||
var _ restic.Counter = (*Counter)(nil)
|
||||
|
||||
// NewCounter starts a new Counter.
|
||||
func NewCounter(interval time.Duration, total uint64, report Func) *Counter {
|
||||
c := new(Counter)
|
||||
@@ -35,16 +39,12 @@ func NewCounter(interval time.Duration, total uint64, report Func) *Counter {
|
||||
|
||||
// Add v to the Counter. This method is concurrency-safe.
|
||||
func (c *Counter) Add(v uint64) {
|
||||
if c != nil {
|
||||
c.value.Add(v)
|
||||
}
|
||||
c.value.Add(v)
|
||||
}
|
||||
|
||||
// SetMax sets the maximum expected counter value. This method is concurrency-safe.
|
||||
func (c *Counter) SetMax(max uint64) {
|
||||
if c != nil {
|
||||
c.max.Store(max)
|
||||
}
|
||||
c.max.Store(max)
|
||||
}
|
||||
|
||||
// Get returns the current value and the maximum of c.
|
||||
@@ -54,7 +54,5 @@ func (c *Counter) Get() (v, max uint64) {
|
||||
}
|
||||
|
||||
func (c *Counter) Done() {
|
||||
if c != nil {
|
||||
c.Updater.Done()
|
||||
}
|
||||
c.Updater.Done()
|
||||
}
|
||||
|
||||
@@ -58,11 +58,3 @@ func TestCounter(t *testing.T) {
|
||||
|
||||
t.Log("number of calls:", ncalls)
|
||||
}
|
||||
|
||||
func TestCounterNil(_ *testing.T) {
|
||||
// Shouldn't panic.
|
||||
var c *progress.Counter
|
||||
c.Add(1)
|
||||
c.SetMax(42)
|
||||
c.Done()
|
||||
}
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
package progress
|
||||
|
||||
import (
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// A Printer can can return a new counter or print messages
|
||||
// at different log levels.
|
||||
// It must be safe to call its methods from concurrent goroutines.
|
||||
type Printer interface {
|
||||
// NewCounter returns a new progress counter. It is not shown if --quiet or --json is specified.
|
||||
NewCounter(description string) *Counter
|
||||
NewCounter(description string) restic.Counter
|
||||
// NewCounterTerminalOnly returns a new progress counter that is only shown if stdout points to a
|
||||
// terminal. It is not shown if --quiet or --json is specified.
|
||||
NewCounterTerminalOnly(description string) *Counter
|
||||
NewCounterTerminalOnly(description string) restic.Counter
|
||||
|
||||
// E reports an error. This message is always printed to stderr.
|
||||
// Appends a newline if not present.
|
||||
@@ -42,12 +46,12 @@ func NewNoopPrinter() Printer {
|
||||
return &noopPrinter{}
|
||||
}
|
||||
|
||||
func (*noopPrinter) NewCounter(_ string) *Counter {
|
||||
return nil
|
||||
func (*noopPrinter) NewCounter(_ string) restic.Counter {
|
||||
return restic.NoopCounter
|
||||
}
|
||||
|
||||
func (*noopPrinter) NewCounterTerminalOnly(_ string) *Counter {
|
||||
return nil
|
||||
func (*noopPrinter) NewCounterTerminalOnly(_ string) restic.Counter {
|
||||
return restic.NoopCounter
|
||||
}
|
||||
|
||||
func (*noopPrinter) E(_ string, _ ...interface{}) {}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
)
|
||||
|
||||
@@ -27,9 +28,9 @@ func CalculateProgressInterval(show bool, json bool, canUpdateStatus bool) time.
|
||||
}
|
||||
|
||||
// newProgressMax returns a progress.Counter that prints to terminal if provided.
|
||||
func newProgressMax(show bool, max uint64, description string, term ui.Terminal) *Counter {
|
||||
func newProgressMax(show bool, max uint64, description string, term ui.Terminal) restic.Counter {
|
||||
if !show {
|
||||
return nil
|
||||
return restic.NoopCounter
|
||||
}
|
||||
interval := CalculateProgressInterval(show, false, term.CanUpdateStatus())
|
||||
|
||||
@@ -57,11 +58,11 @@ type terminalPrinter struct {
|
||||
v uint
|
||||
}
|
||||
|
||||
func (t *terminalPrinter) NewCounter(description string) *Counter {
|
||||
func (t *terminalPrinter) NewCounter(description string) restic.Counter {
|
||||
return newProgressMax(t.v > 0, 0, description, t.term)
|
||||
}
|
||||
|
||||
func (t *terminalPrinter) NewCounterTerminalOnly(description string) *Counter {
|
||||
func (t *terminalPrinter) NewCounterTerminalOnly(description string) restic.Counter {
|
||||
return newProgressMax(t.v > 0 && t.term.OutputIsTerminal(), 0, description, t.term)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user