restic: introduce Counter interface to decouple from ui/progress (#21861)

decouple restic and ui/progress packages

internal/repository/checker.go

@@ -16,7 +16,6 @@ import (
 	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/progress"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -244,7 +243,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
 }
 
 // ReadPacks loads data from specified packs and checks the integrity.
-func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p *progress.Counter, errChan chan<- error) {
+func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID]int64) map[restic.ID]int64, p restic.Counter, errChan chan<- error) {
 	defer close(errChan)
 
 	// compute pack size using index entries

internal/repository/checker_test.go

@@ -36,7 +36,7 @@ func TestGapInBlobs(t *testing.T) {
 	repo, _, cleanup := TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	err := repo.LoadIndex(context.TODO(), nil)
+	err := repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
 	rtest.OK(t, err)
 
 	repoPacks, err := pack.Size(context.TODO(), repo, false)
@@ -90,7 +90,7 @@ func runReadPacks(chkr *Checker) []error {
 		func(ctx context.Context, errCh chan<- error) {
 			chkr.ReadPacks(ctx, func(packs map[restic.ID]int64) map[restic.ID]int64 {
 				return packs
-			}, nil, errCh)
+			}, restic.NoopCounter, errCh)
 		})
 }
 
@@ -162,7 +162,7 @@ func setupChecker(t *testing.T, wrap func(backend.Backend) backend.Backend) *Che
 	chkr := newChecker(checkRepo)
 
 	// make sure the index is loaded
-	err := checkRepo.LoadIndex(context.TODO(), nil)
+	err := checkRepo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
 	rtest.OK(t, err)
 
 	return chkr

internal/repository/index/master_index.go

@@ -10,7 +10,6 @@ import (
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/progress"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -281,7 +280,7 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
 	return nil
 }
 
-func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
+func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p restic.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
 	indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
 	if err != nil {
 		return err
@@ -290,31 +289,27 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
 	if err != nil {
 		return err
 	}
-	if p != nil {
-		var numIndexFiles uint64
-		err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
-			if loadedIDs.Has(id) {
-				// skip already loaded indexes
-				return nil
-			}
-			numIndexFiles++
+	var numIndexFiles uint64
+	err = indexList.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
+		if loadedIDs.Has(id) {
+			// skip already loaded indexes
 			return nil
-		})
-		if err != nil {
-			return err
 		}
-		p.SetMax(numIndexFiles)
-		defer p.Done()
+		numIndexFiles++
+		return nil
+	})
+	if err != nil {
+		return err
 	}
+	p.SetMax(numIndexFiles)
+	defer p.Done()
 
 	err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error {
 		if loadedIDs.Has(id) {
 			// skip already loaded indexes
 			return nil
 		}
-		if p != nil {
-			p.Add(1)
-		}
+		p.Add(1)
 		if cb != nil {
 			err = cb(id, idx, err)
 		}
@@ -368,8 +363,8 @@ func (mi *MasterIndex) prepareIncrementalLoad(ctx context.Context, indexList res
 }
 
 type MasterIndexRewriteOpts struct {
-	SaveProgress   *progress.Counter
-	DeleteProgress func() *progress.Counter
+	SaveProgress   restic.Counter
+	DeleteProgress func() restic.Counter
 	DeleteReport   func(id restic.ID, err error)
 }
 
@@ -396,6 +391,9 @@ func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked[restic.
 	}
 
 	p := opts.SaveProgress
+	if p == nil {
+		p = restic.NoopCounter
+	}
 	p.SetMax(uint64(len(indexes)))
 
 	// reset state which is not necessary for Rewrite and just consumes a lot of memory
@@ -553,7 +551,7 @@ func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked[restic.
 		return fmt.Errorf("failed to rewrite indexes: %w", err)
 	}
 
-	p = nil
+	p = restic.NoopCounter
 	if opts.DeleteProgress != nil {
 		p = opts.DeleteProgress()
 	}
@@ -571,7 +569,7 @@ func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked[restic.
 // It is only intended for use by prune with the UnsafeRecovery option.
 //
 // Must not be called concurrently to any other MasterIndex operation.
-func (mi *MasterIndex) SaveFallback(ctx context.Context, repo restic.SaverRemoverUnpacked[restic.FileType], excludePacks restic.IDSet, p *progress.Counter) error {
+func (mi *MasterIndex) SaveFallback(ctx context.Context, repo restic.SaverRemoverUnpacked[restic.FileType], excludePacks restic.IDSet, p restic.Counter) error {
 	p.SetMax(uint64(len(mi.Packs(excludePacks))))
 
 	mi.idxMutex.Lock()

internal/repository/index/master_index_test.go

@@ -438,18 +438,18 @@ func testIndexSave(t *testing.T, version uint) {
 			return idx.Rewrite(context.TODO(), repo, nil, restic.NewIDSet(), nil, index.MasterIndexRewriteOpts{})
 		}},
 		{"SaveFallback", func(idx *index.MasterIndex, repo restic.Unpacked[restic.FileType]) error {
-			err := restic.ParallelRemove(context.TODO(), repo, idx.IDs(), restic.IndexFile, nil, nil)
+			err := restic.ParallelRemove(context.TODO(), repo, idx.IDs(), restic.IndexFile, nil, restic.NoopCounter)
 			if err != nil {
 				return nil
 			}
-			return idx.SaveFallback(context.TODO(), repo, restic.NewIDSet(), nil)
+			return idx.SaveFallback(context.TODO(), repo, restic.NewIDSet(), restic.NoopCounter)
 		}},
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			repo, unpacked := createFilledRepo(t, 3, version)
 
 			idx := index.NewMasterIndex()
-			rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
+			rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
 			blobs := make(map[pack.PackedBlob]struct{})
 			for pb := range idx.Values() {
 				blobs[*pb] = struct{}{}
@@ -457,7 +457,7 @@ func testIndexSave(t *testing.T, version uint) {
 
 			rtest.OK(t, test.saver(idx, unpacked))
 			idx = index.NewMasterIndex()
-			rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
+			rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
 
 			for pb := range idx.Values() {
 				if _, ok := blobs[*pb]; ok {
@@ -482,7 +482,7 @@ func testIndexSavePartial(t *testing.T, version uint) {
 
 	// capture blob list before adding fourth snapshot
 	idx := index.NewMasterIndex()
-	rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
+	rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
 	blobs := make(map[pack.PackedBlob]struct{})
 	for pb := range idx.Values() {
 		blobs[*pb] = struct{}{}
@@ -497,12 +497,12 @@ func testIndexSavePartial(t *testing.T, version uint) {
 
 	// rewrite index and remove pack files of new snapshot
 	idx = index.NewMasterIndex()
-	rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
+	rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
 	rtest.OK(t, idx.Rewrite(context.TODO(), unpacked, newPacks, nil, nil, index.MasterIndexRewriteOpts{}))
 
 	// check blobs
 	idx = index.NewMasterIndex()
-	rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil))
+	rtest.OK(t, idx.Load(context.TODO(), repo, restic.NoopCounter, nil))
 	for pb := range idx.Values() {
 		if _, ok := blobs[*pb]; ok {
 			delete(blobs, *pb)
@@ -513,7 +513,7 @@ func testIndexSavePartial(t *testing.T, version uint) {
 	rtest.Equals(t, 0, len(blobs), "saved index is missing blobs")
 
 	// remove pack files to make check happy
-	rtest.OK(t, restic.ParallelRemove(context.TODO(), unpacked, newPacks, restic.PackFile, nil, nil))
+	rtest.OK(t, restic.ParallelRemove(context.TODO(), unpacked, newPacks, restic.PackFile, nil, restic.NoopCounter))
 
 	checker.TestCheckRepo(t, repo)
 }
@@ -620,14 +620,14 @@ func TestRewriteOversizedIndex(t *testing.T) {
 
 	// construct master index for the oversized index
 	mi := index.NewMasterIndex()
-	rtest.OK(t, mi.Load(context.Background(), repo, nil, nil))
+	rtest.OK(t, mi.Load(context.Background(), repo, restic.NoopCounter, nil))
 
 	// rewrite the index
 	rtest.OK(t, mi.Rewrite(context.Background(), unpacked, nil, nil, nil, index.MasterIndexRewriteOpts{}))
 
 	// load the rewritten indexes
 	mi2 := index.NewMasterIndex()
-	rtest.OK(t, mi2.Load(context.Background(), repo, nil, nil))
+	rtest.OK(t, mi2.Load(context.Background(), repo, restic.NoopCounter, nil))
 
 	// verify that blobs are still in the index
 	for _, blob := range blobs {
@@ -685,7 +685,7 @@ func TestRewriteSplitPacks(t *testing.T) {
 	rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, restic.NewIDSet(blobOther.PackID()), nil, nil, index.MasterIndexRewriteOpts{}))
 
 	mi = index.NewMasterIndex()
-	rtest.OK(t, mi.Load(context.TODO(), repo, nil, nil))
+	rtest.OK(t, mi.Load(context.TODO(), repo, restic.NoopCounter, nil))
 
 	// test that all blobs are still in the index
 	for _, blob := range []*pack.PackedBlob{blob1, blob2} {
@@ -746,7 +746,7 @@ func TestRewriteFullPacks(t *testing.T) {
 	rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
 
 	mi2 := index.NewMasterIndex()
-	rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
+	rtest.OK(t, mi2.Load(context.TODO(), repo, restic.NoopCounter, nil))
 
 	afterRewrite := mi2.IDs()
 	rtest.Equals(t, 2, len(afterRewrite))

internal/repository/index_list_test.go

@@ -23,7 +23,7 @@ func TestAllIndexBlobs(t *testing.T) {
 		return nil
 	}))
 
-	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
+	rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
 
 	fromMaster := restic.NewBlobSet()
 	rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackBlob) {

internal/repository/prune_test.go

@@ -153,7 +153,7 @@ func TestPruneSmall(t *testing.T) {
 
 	// and reopen repository with default packsize
 	repo = repository.TestOpenBackend(t, be)
-	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
+	rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
 
 	opts := repository.PruneOptions{
 		MaxRepackBytes: math.MaxUint64,

internal/repository/repack.go

@@ -10,7 +10,6 @@ import (
 	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/progress"
 
 	"golang.org/x/sync/errgroup"
 )
@@ -37,7 +36,7 @@ func CopyBlobs(
 	dstUploader restic.BlobSaverWithAsync,
 	packs restic.IDSet,
 	keepBlobs repackBlobSet,
-	p *progress.Counter,
+	p restic.Counter,
 	logf LogFunc,
 ) error {
 	debug.Log("repacking %d packs while keeping %d blobs", len(packs), keepBlobs.Len())
@@ -62,7 +61,7 @@ func repack(
 	uploader restic.BlobSaverWithAsync,
 	packs restic.IDSet,
 	keepBlobs repackBlobSet,
-	p *progress.Counter,
+	p restic.Counter,
 	logf LogFunc,
 ) error {
 

internal/repository/repack_test.go

@@ -150,7 +150,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
 
 func repack(t *testing.T, repo *repository.Repository, be backend.Backend, packs restic.IDSet, blobs restic.BlobSet) {
 	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, nil, nil)
+		return repository.CopyBlobs(ctx, repo, repo, uploader, packs, blobs, restic.NoopCounter, nil)
 	}))
 
 	for id := range packs {
@@ -163,7 +163,7 @@ func rebuildAndReloadIndex(t *testing.T, repo *repository.Repository) {
 		ReadAllPacks: true,
 	}, progress.NewNoopPrinter()))
 
-	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
+	rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
 }
 
 func TestRepack(t *testing.T) {
@@ -272,7 +272,7 @@ func testRepackCopy(t *testing.T, version uint) {
 	copyPacks := findPacksForBlobs(t, repo, keepBlobs)
 
 	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		return repository.CopyBlobs(ctx, repo, dstRepo, uploader, copyPacks, keepBlobs, nil, nil)
+		return repository.CopyBlobs(ctx, repo, dstRepo, uploader, copyPacks, keepBlobs, restic.NoopCounter, nil)
 	}))
 	rebuildAndReloadIndex(t, dstRepo)
 
@@ -310,7 +310,7 @@ func testRepackWrongBlob(t *testing.T, version uint) {
 	rewritePacks := findPacksForBlobs(t, repo, keepBlobs)
 
 	err := repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, nil, nil)
+		return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, restic.NoopCounter, nil)
 	})
 	if err == nil {
 		t.Fatal("expected repack to fail but got no error")
@@ -359,7 +359,7 @@ func testRepackBlobFallback(t *testing.T, version uint) {
 
 	// repack must fallback to valid copy
 	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, nil, nil)
+		return repository.CopyBlobs(ctx, repo, repo, uploader, rewritePacks, keepBlobs, restic.NoopCounter, nil)
 	}))
 
 	keepBlobs = restic.NewBlobSet(restic.BlobHandle{Type: restic.DataBlob, ID: id})

internal/repository/repair_index.go

@@ -32,7 +32,7 @@ func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions,
 
 	} else {
 		printer.P("loading indexes...\n")
-		err := repo.loadIndexWithCallback(ctx, nil, func(id restic.ID, _ *index.Index, err error) error {
+		err := repo.loadIndexWithCallback(ctx, restic.NoopTerminalCounterFactory, func(id restic.ID, _ *index.Index, err error) error {
 			if err != nil {
 				printer.E("removing invalid index %v: %v\n", id, err)
 				obsoleteIndexes = append(obsoleteIndexes, id)
@@ -109,7 +109,7 @@ func rewriteIndexFiles(ctx context.Context, repo *Repository, removePacks restic
 	bar := printer.NewCounter("indexes processed")
 	return repo.idx.Rewrite(ctx, &internalRepository{repo}, removePacks, oldIndexes, extraObsolete, index.MasterIndexRewriteOpts{
 		SaveProgress: bar,
-		DeleteProgress: func() *progress.Counter {
+		DeleteProgress: func() restic.Counter {
 			return printer.NewCounter("old indexes deleted")
 		},
 		DeleteReport: func(id restic.ID, err error) {

internal/repository/repair_pack_test.go

@@ -131,7 +131,7 @@ func testRepairBrokenPack(t *testing.T, version uint) {
 
 			rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, progress.NewNoopPrinter()))
 			// reload index
-			rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
+			rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
 
 			packsAfter := listPacks(t, repo)
 			blobsAfter := listBlobs(repo)

internal/repository/repository.go

@@ -20,7 +20,6 @@ import (
 	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/progress"
 
 	"golang.org/x/sync/errgroup"
 )
@@ -716,10 +715,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p restic.TerminalCounterFact
 func (r *Repository) loadIndexWithCallback(ctx context.Context, p restic.TerminalCounterFactory, cb func(id restic.ID, idx *index.Index, err error) error) error {
 	debug.Log("Loading index")
 
-	var bar *progress.Counter
-	if p != nil {
-		bar = p.NewCounterTerminalOnly("index files loaded")
-	}
+	bar := p.NewCounterTerminalOnly("index files loaded")
 
 	err := r.idx.Load(ctx, r, bar, cb)
 	if err != nil {
@@ -758,7 +754,7 @@ func (r *Repository) loadIndexWithCallback(ctx context.Context, p restic.Termina
 // createIndexFromPacks creates a new index by reading all given pack files (with sizes).
 // The index is added to the MasterIndex but not marked as finalized.
 // Returned is the list of pack files which could not be read.
-func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[restic.ID]int64, p *progress.Counter) (invalid restic.IDs, err error) {
+func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[restic.ID]int64, p restic.Counter) (invalid restic.IDs, err error) {
 	var m sync.Mutex
 
 	debug.Log("Loading index from pack files")

internal/repository/repository_test.go

@@ -322,7 +322,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
 	repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
 	defer cleanup()
 
-	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
+	rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
 }
 
 // loadIndex loads the index id from backend and returns it.
@@ -380,7 +380,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
 	rtest.OK(t, err)
 	repo := repository.TestOpenBackend(t, &damageOnceBackend{Backend: be})
 
-	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
+	rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
 }
 
 // saveRandomDataBlobs generates random data blobs and saves them to the repository.

internal/repository/testing.go

@@ -167,7 +167,7 @@ func TestNewLock(_ *testing.T, repo *Repository, exclusive bool) (*restic.Lock,
 func TestCheckRepo(t testing.TB, repo *Repository) {
 	chkr := newChecker(repo)
 
-	hints, errs := chkr.LoadIndex(context.TODO(), nil)
+	hints, errs := chkr.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
 	if len(errs) != 0 {
 		t.Fatalf("errors loading index: %v", errs)
 	}
@@ -188,7 +188,7 @@ func TestCheckRepo(t testing.TB, repo *Repository) {
 	errChan = make(chan error)
 	go chkr.ReadPacks(context.TODO(), func(packs map[restic.ID]int64) map[restic.ID]int64 {
 		return packs
-	}, nil, errChan)
+	}, restic.NoopCounter, errChan)
 
 	for err := range errChan {
 		t.Error(err)