amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-06-05 08:29:44 +00:00
Code Issues Projects Releases Wiki Activity

Show progress for restic stats (#5705)

Browse Source 
Co-authored-by: Paulo Saraiva <paulo.saraiva@cern.ch>
This commit is contained in:
Paulo Saraiva
2026-05-15 20:31:20 +02:00
committed by GitHub
parent 8abbc3703d
commit 6de6cc6c75
3 changed files with 124 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/restic/cmd_stats.go
+88 -6
View File
@@ -7,6 +7,8 @@ import (
"fmt"
"path/filepath"
"strings"
"sync"
"time"
"github.com/restic/chunker"
"github.com/restic/restic/internal/crypto"
@@ -134,8 +136,21 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
SnapshotsCount: 0,
}
var snapshots data.Snapshots
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args, printer) {
err = statsWalkSnapshot(ctx, sn, repo, opts, stats)
snapshots = append(snapshots, sn)
}
statsProgress := newStatsProgress(term, uint64(len(snapshots)))
updater := progress.NewUpdater(ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()), func(runtime time.Duration, final bool) {
statsProgress.printProgress(runtime, final)
})
defer updater.Done()
for _, sn := range snapshots {
err = statsWalkSnapshot(ctx, sn, repo, opts, stats, statsProgress)
if err != nil {
return fmt.Errorf("error walking snapshot: %v", err)
}
@@ -160,6 +175,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
}
}
stats.TotalBlobCount++
statsProgress.update(0, 1, uint64(pbs[0].Length))
}
if stats.TotalCompressedBlobsSize > 0 {
stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize)
@@ -203,7 +219,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
return nil
}
func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error {
func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer, progress *statsProgress) error {
progress.processSnapshot()
if snapshot.Tree == nil {
return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
}
@@ -218,7 +235,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic
hardLinkIndex := restorer.NewHardlinkIndex[struct{}]()
err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{
ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex),
ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex, progress),
})
if err != nil {
return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
@@ -227,7 +244,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *data.Snapshot, repo restic
return nil
}
func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc {
func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}], progress *statsProgress) walker.WalkFunc {
return func(parentTreeID restic.ID, npath string, node *data.Node, nodeErr error) error {
if nodeErr != nil {
return nodeErr
@@ -235,7 +252,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
if node == nil {
return nil
}
progress.update(1, 0, uint64(node.Size))
if opts.countMode == countModeUniqueFilesByContents || opts.countMode == countModeBlobsPerFile {
// only count this file if we haven't visited it before
fid := makeFileIDByContents(node)
@@ -255,6 +272,7 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
// ensure we have this file (by path) in our map; in this
// mode, a file is unique by both contents and path
nodePath := filepath.Join(npath, node.Name)
progress.update(0, 1, 0)
if _, ok := stats.fileBlobs[nodePath]; !ok {
stats.fileBlobs[nodePath] = restic.NewIDSet()
stats.TotalFileCount++
@@ -294,7 +312,6 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
}
}
}
return nil
}
}
@@ -352,6 +369,71 @@ type statsContainer struct {
// independent of references to files
blobs restic.AssociatedBlobSet
}
type statsProgress struct {
term ui.Terminal
m sync.Mutex
snapshotCount uint64
processedSnapshotCount uint64
processedFileCount uint64
processedBlobCount uint64
processedSize uint64
}
func newStatsProgress(term ui.Terminal, snapshotCount uint64) *statsProgress {
return &statsProgress{
term: term,
snapshotCount: snapshotCount,
}
}
func (s *statsProgress) printProgress(runtime time.Duration, final bool) {
s.m.Lock()
progressBase := s.processedSnapshotCount
if progressBase > 0 && !final {
progressBase--
}
status := fmt.Sprintf("[%s] %s %d / %d snapshots", ui.FormatDuration(runtime), ui.FormatPercent(progressBase, s.snapshotCount), s.processedSnapshotCount, s.snapshotCount)
if s.processedFileCount > 0 {
status += fmt.Sprintf(", %v files", s.processedFileCount)
}
if s.processedBlobCount > 0 {
status += fmt.Sprintf(", %d blobs", s.processedBlobCount)
}
status += fmt.Sprintf(", %s", ui.FormatBytes(s.processedSize))
s.m.Unlock()
if final {
s.term.SetStatus(nil)
s.term.Print(status)
} else {
s.term.SetStatus([]string{status})
}
}
func (s *statsProgress) update(fileCount uint64, blobCount uint64, size uint64) {
s.m.Lock()
defer s.m.Unlock()
s.processedFileCount += fileCount
s.processedBlobCount += blobCount
s.processedSize += size
}
func (s *statsProgress) processSnapshot() {
s.m.Lock()
defer s.m.Unlock()
s.processedSnapshotCount++
s.processedFileCount = 0
s.processedBlobCount = 0
s.processedSize = 0
}
// fileID is a 256-bit hash that distinguishes unique files.
type fileID [32]byte
cmd/restic/cmd_stats_test.go
+28
View File
@@ -2,8 +2,10 @@ package main
import (
"testing"
"time"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
)
func TestSizeHistogramNew(t *testing.T) {
@@ -60,3 +62,29 @@ func TestSizeHistogramString(t *testing.T) {
rtest.Equals(t, "Count: 3\nTotal Size: 11 B\nSize Count\n-------------------\n 0 - 0 Byte 1\n 1 - 9 Byte 1\n10 - 42 Byte 1\n-------------------\n", h.String())
})
}
func TestStatsProgress(t *testing.T) {
term := &ui.MockTerminal{}
progress := newStatsProgress(term, 2)
progress.printProgress(0*time.Second, false)
rtest.Equals(t, []string{"[0:00] 0.00% 0 / 2 snapshots, 0 B"}, term.Output)
progress.processSnapshot()
progress.update(1, 2, 3)
progress.printProgress(5*time.Second, false)
// Output differs from the previous one because the progress is based on the number of processed snapshots,
// 1/2 snapshots means processing the snapshot 1 currently
rtest.Equals(t, []string{"[0:05] 0.00% 1 / 2 snapshots, 1 files, 2 blobs, 3 B"}, term.Output)
progress.processSnapshot()
progress.printProgress(10*time.Second, false)
rtest.Equals(t, []string{"[0:10] 50.00% 2 / 2 snapshots, 0 B"}, term.Output)
progress.update(4, 5, 6)
progress.printProgress(15*time.Second, false)
rtest.Equals(t, []string{"[0:15] 50.00% 2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output)
progress.printProgress(20*time.Second, true)
rtest.Equals(t, []string{"[0:20] 100.00% 2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output)
}