diff --git a/changelog/unreleased/issue-5689 b/changelog/unreleased/issue-5689 new file mode 100644 index 000000000..1c2001493 --- /dev/null +++ b/changelog/unreleased/issue-5689 @@ -0,0 +1,8 @@ +Enhancement: Show progress bar for `restic stats` + +Previously, running `restic stats` would only give progress updates for loading the index. +Now it displays progress for how many snapshots, files, and blobs were processed so far. +This lets users better understand if the command is working as expected or where it is hanging. + +https://github.com/restic/restic/issues/5689 +https://github.com/restic/restic/pull/5705 \ No newline at end of file diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index ac4e860aa..fdf33f074 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -7,6 +7,8 @@ import ( "fmt" "path/filepath" "strings" + "sync" + "time" "github.com/restic/chunker" "github.com/restic/restic/internal/crypto" @@ -134,8 +136,21 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args SnapshotsCount: 0, } + var snapshots data.Snapshots for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args, printer) { - err = statsWalkSnapshot(ctx, sn, repo, opts, stats) + snapshots = append(snapshots, sn) + } + + statsProgress := newStatsProgress(term, uint64(len(snapshots))) + + updater := progress.NewUpdater(ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()), func(runtime time.Duration, final bool) { + statsProgress.printProgress(runtime, final) + }) + + defer updater.Done() + + for _, sn := range snapshots { + err = statsWalkSnapshot(ctx, sn, repo, opts, stats, statsProgress) if err != nil { return fmt.Errorf("error walking snapshot: %v", err) } @@ -160,6 +175,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args } } stats.TotalBlobCount++ + statsProgress.update(0, 1, uint64(pbs[0].Length)) } if stats.TotalCompressedBlobsSize > 0 { stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize) @@ -203,7 +219,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args return nil } -func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error { +func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer, progress *statsProgress) error { + progress.processSnapshot() if snapshot.Tree == nil { return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str()) } @@ -218,7 +235,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{ - ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex), + ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex, progress), }) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) @@ -227,7 +244,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic return nil } -func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { +func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}], progress *statsProgress) walker.WalkFunc { return func(parentTreeID restic.ID, npath string, node *data.Node, nodeErr error) error { if nodeErr != nil { return nodeErr @@ -235,7 +252,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, if node == nil { return nil } - + progress.update(1, 0, uint64(node.Size)) if opts.countMode == countModeUniqueFilesByContents || opts.countMode == countModeBlobsPerFile { // only count this file if we haven't visited it before fid := makeFileIDByContents(node) @@ -255,6 +272,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, // ensure we have this file (by path) in our map; in this // mode, a file is unique by both contents and path nodePath := filepath.Join(npath, node.Name) + progress.update(0, 1, 0) if _, ok := stats.fileBlobs[nodePath]; !ok { stats.fileBlobs[nodePath] = restic.NewIDSet() stats.TotalFileCount++ @@ -294,7 +312,6 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, } } } - return nil } } @@ -352,6 +369,71 @@ type statsContainer struct { // independent of references to files blobs restic.AssociatedBlobSet } +type statsProgress struct { + term ui.Terminal + m sync.Mutex + snapshotCount uint64 + + processedSnapshotCount uint64 + processedFileCount uint64 + processedBlobCount uint64 + processedSize uint64 +} + +func newStatsProgress(term ui.Terminal, snapshotCount uint64) *statsProgress { + return &statsProgress{ + term: term, + snapshotCount: snapshotCount, + } +} + +func (s *statsProgress) printProgress(runtime time.Duration, final bool) { + s.m.Lock() + + progressBase := s.processedSnapshotCount + if progressBase > 0 && !final { + progressBase-- + } + + status := fmt.Sprintf("[%s] %s %d / %d snapshots", ui.FormatDuration(runtime), ui.FormatPercent(progressBase, s.snapshotCount), s.processedSnapshotCount, s.snapshotCount) + + if s.processedFileCount > 0 { + status += fmt.Sprintf(", %v files", s.processedFileCount) + } + + if s.processedBlobCount > 0 { + status += fmt.Sprintf(", %d blobs", s.processedBlobCount) + } + + status += fmt.Sprintf(", %s", ui.FormatBytes(s.processedSize)) + s.m.Unlock() + + if final { + s.term.SetStatus(nil) + s.term.Print(status) + } else { + s.term.SetStatus([]string{status}) + } +} + +func (s *statsProgress) update(fileCount uint64, blobCount uint64, size uint64) { + s.m.Lock() + defer s.m.Unlock() + + s.processedFileCount += fileCount + s.processedBlobCount += blobCount + s.processedSize += size +} + +func (s *statsProgress) processSnapshot() { + s.m.Lock() + defer s.m.Unlock() + + s.processedSnapshotCount++ + s.processedFileCount = 0 + s.processedBlobCount = 0 + s.processedSize = 0 +} // fileID is a 256-bit hash that distinguishes unique files. type fileID [32]byte diff --git a/cmd/restic/cmd_stats_test.go b/cmd/restic/cmd_stats_test.go index 02d37acd9..231bd65ff 100644 --- a/cmd/restic/cmd_stats_test.go +++ b/cmd/restic/cmd_stats_test.go @@ -2,8 +2,10 @@ package main import ( "testing" + "time" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" ) func TestSizeHistogramNew(t *testing.T) { @@ -60,3 +62,29 @@ func TestSizeHistogramString(t *testing.T) { rtest.Equals(t, "Count: 3\nTotal Size: 11 B\nSize Count\n-------------------\n 0 - 0 Byte 1\n 1 - 9 Byte 1\n10 - 42 Byte 1\n-------------------\n", h.String()) }) } + +func TestStatsProgress(t *testing.T) { + term := &ui.MockTerminal{} + + progress := newStatsProgress(term, 2) + progress.printProgress(0*time.Second, false) + rtest.Equals(t, []string{"[0:00] 0.00% 0 / 2 snapshots, 0 B"}, term.Output) + + progress.processSnapshot() + progress.update(1, 2, 3) + progress.printProgress(5*time.Second, false) + // Output differs from the previous one because the progress is based on the number of processed snapshots, + // 1/2 snapshots means processing the snapshot 1 currently + rtest.Equals(t, []string{"[0:05] 0.00% 1 / 2 snapshots, 1 files, 2 blobs, 3 B"}, term.Output) + + progress.processSnapshot() + progress.printProgress(10*time.Second, false) + rtest.Equals(t, []string{"[0:10] 50.00% 2 / 2 snapshots, 0 B"}, term.Output) + + progress.update(4, 5, 6) + progress.printProgress(15*time.Second, false) + rtest.Equals(t, []string{"[0:15] 50.00% 2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output) + + progress.printProgress(20*time.Second, true) + rtest.Equals(t, []string{"[0:20] 100.00% 2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output) +}