From 0f05277b47d8e36ce35ac846dd0ab8ca5291858e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 19 Nov 2025 21:39:11 +0100 Subject: [PATCH 1/8] index: add sub and intersect method to AssociatedSet --- internal/repository/index/associated_data.go | 34 ++++++++ .../repository/index/associated_data_test.go | 78 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/internal/repository/index/associated_data.go b/internal/repository/index/associated_data.go index ad9b3288e..1267bdc4f 100644 --- a/internal/repository/index/associated_data.go +++ b/internal/repository/index/associated_data.go @@ -108,6 +108,40 @@ func (a *AssociatedSet[T]) Delete(bh restic.BlobHandle) { } } +type haser interface { + Has(bh restic.BlobHandle) bool +} + +// Intersect returns a new set containing the handles that are present in both sets. +func (a *AssociatedSet[T]) Intersect(other haser) *AssociatedSet[T] { + result := NewAssociatedSet[T](a.idx) + // Determining the smaller set already requires iterating over all keys + // and thus provides no performance benefit. + for bh := range a.Keys() { + if other.Has(bh) { + // preserve value receiver + val, _ := a.Get(bh) + result.Set(bh, val) + } + } + + return result +} + +// Sub returns a new set containing all handles that are present in a but not in +// other. +func (a *AssociatedSet[T]) Sub(other haser) *AssociatedSet[T] { + result := NewAssociatedSet[T](a.idx) + for bh := range a.Keys() { + if !other.Has(bh) { + val, _ := a.Get(bh) + result.Set(bh, val) + } + } + + return result +} + func (a *AssociatedSet[T]) Len() int { count := 0 for range a.All() { diff --git a/internal/repository/index/associated_data_test.go b/internal/repository/index/associated_data_test.go index 6d70b3dff..2d4611f5c 100644 --- a/internal/repository/index/associated_data_test.go +++ b/internal/repository/index/associated_data_test.go @@ -157,3 +157,81 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) { test.Equals(t, list(bs), restic.BlobHandles(nil)) test.Equals(t, 0, len(bs.overflow)) } + +func TestAssociatedSetIntersectAndSub(t *testing.T) { + mi := NewMasterIndex() + saver := &noopSaver{} + + bh1, blob1 := makeFakePackedBlob() + bh2, blob2 := makeFakePackedBlob() + bh3, blob3 := makeFakePackedBlob() + bh4, blob4 := makeFakePackedBlob() + + test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, []restic.Blob{blob1.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, []restic.Blob{blob2.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, []restic.Blob{blob3.Blob}, saver)) + test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, []restic.Blob{blob4.Blob}, saver)) + test.OK(t, mi.Flush(context.TODO(), saver)) + + t.Run("Intersect", func(t *testing.T) { + bs1, bs2 := NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + test.Equals(t, bs1.Intersect(bs2).Len(), 0) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 10) + bs2.Set(bh2, 20) + test.Equals(t, bs1.Intersect(bs2).Len(), 0) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh3, 40) + bs2.Set(bh3, 50) + bs2.Set(bh4, 60) + result := bs1.Intersect(bs2) + test.Equals(t, result.Len(), 1) + val, _ := result.Get(bh3) + test.Equals(t, uint8(40), val) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh3, 40) + bs1.Set(bh4, 70) + bs2.Set(bh3, 50) + bs2.Set(bh4, 60) + result = bs1.Intersect(bs2) + test.Equals(t, result.Len(), 2) + val, _ = result.Get(bh3) + test.Equals(t, uint8(40), val) + val, _ = result.Get(bh4) + test.Equals(t, uint8(70), val) + }) + + t.Run("Sub", func(t *testing.T) { + bs1, bs2 := NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + test.Equals(t, bs1.Sub(bs2).Len(), 0) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 10) + bs1.Set(bh2, 20) + bs2.Set(bh3, 30) + result := bs1.Sub(bs2) + test.Equals(t, result.Len(), 2) + val, _ := result.Get(bh1) + test.Equals(t, uint8(10), val) + val, _ = result.Get(bh2) + test.Equals(t, uint8(20), val) + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 10) + bs1.Set(bh2, 20) + bs1.Set(bh3, 40) + bs2.Set(bh2, 50) + result = bs1.Sub(bs2) + test.Equals(t, result.Len(), 2) + test.Assert(t, result.Has(bh1) && result.Has(bh3) && !result.Has(bh2), "only bh1 and bh3 should be in result") + + bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi) + bs1.Set(bh1, 60) + bs2.Set(bh1, 70) + bs2.Set(bh2, 80) + test.Equals(t, bs1.Sub(bs2).Len(), 0) + }) +} From 07d090f2333a6ee87ca135dc0deede6dae501543 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 19 Nov 2025 21:39:48 +0100 Subject: [PATCH 2/8] repository: expose AssociatedBlobSet via repository interface --- internal/repository/repository.go | 16 ++++++++++++++++ internal/restic/repository.go | 12 ++++++++++++ 2 files changed, 28 insertions(+) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 3b26b1f90..9abdaeec2 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -787,6 +787,22 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest return invalid, nil } +func (r *Repository) NewAssociatedBlobSet() restic.AssociatedBlobSet { + return &associatedBlobSet{*index.NewAssociatedSet[struct{}](r.idx)} +} + +// associatedBlobSet is a wrapper around index.AssociatedSet to implement the restic.AssociatedBlobSet interface. +type associatedBlobSet struct { + index.AssociatedSet[struct{}] +} + +func (s *associatedBlobSet) Intersect(other restic.AssociatedBlobSet) restic.AssociatedBlobSet { + return &associatedBlobSet{*s.AssociatedSet.Intersect(other)} +} +func (s *associatedBlobSet) Sub(other restic.AssociatedBlobSet) restic.AssociatedBlobSet { + return &associatedBlobSet{*s.AssociatedSet.Sub(other)} +} + // prepareCache initializes the local cache. indexIDs is the list of IDs of // index files still present in the repo. func (r *Repository) prepareCache() error { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 2f1373641..ed0c64cf0 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -2,6 +2,7 @@ package restic import ( "context" + "iter" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" @@ -26,6 +27,7 @@ type Repository interface { LookupBlob(t BlobType, id ID) []PackedBlob LookupBlobSize(t BlobType, id ID) (size uint, exists bool) + NewAssociatedBlobSet() AssociatedBlobSet // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. ListBlobs(ctx context.Context, fn func(PackedBlob)) error @@ -186,3 +188,13 @@ type FindBlobSet interface { Has(bh BlobHandle) bool Insert(bh BlobHandle) } + +type AssociatedBlobSet interface { + Has(bh BlobHandle) bool + Insert(bh BlobHandle) + Delete(bh BlobHandle) + Len() int + Keys() iter.Seq[BlobHandle] + Intersect(other AssociatedBlobSet) AssociatedBlobSet + Sub(other AssociatedBlobSet) AssociatedBlobSet +} From ff099a216a169b4b53af3511305c0ed704e0a98d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 19 Nov 2025 21:40:36 +0100 Subject: [PATCH 3/8] copy: use AssociatedBlobSet --- cmd/restic/cmd_copy.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index bf86de8a7..823dcaf4e 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -252,7 +252,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep return visited }, nil) - copyBlobs := restic.NewBlobSet() + copyBlobs := srcRepo.NewAssociatedBlobSet() packList := restic.NewIDSet() enqueue := func(h restic.BlobHandle) { @@ -299,11 +299,11 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep } // copyStats: print statistics for the blobs to be copied -func copyStats(srcRepo restic.Repository, copyBlobs restic.BlobSet, packList restic.IDSet, printer progress.Printer) uint64 { +func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, packList restic.IDSet, printer progress.Printer) uint64 { // count and size countBlobs := 0 sizeBlobs := uint64(0) - for blob := range copyBlobs { + for blob := range copyBlobs.Keys() { for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) { countBlobs++ sizeBlobs += uint64(blob.Length) From d91fe1d7e1beee30ca06ea319c6114f62dae13d0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 19 Nov 2025 21:40:48 +0100 Subject: [PATCH 4/8] diff: use AssociatedBlobSet --- cmd/restic/cmd_diff.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 3bd8e37b8..58138e9e5 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -124,7 +124,7 @@ func (s *DiffStat) Add(node *data.Node) { } // addBlobs adds the blobs of node to s. -func addBlobs(bs restic.BlobSet, node *data.Node) { +func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) { if node == nil { return } @@ -148,18 +148,18 @@ func addBlobs(bs restic.BlobSet, node *data.Node) { } type DiffStatsContainer struct { - MessageType string `json:"message_type"` // "statistics" - SourceSnapshot string `json:"source_snapshot"` - TargetSnapshot string `json:"target_snapshot"` - ChangedFiles int `json:"changed_files"` - Added DiffStat `json:"added"` - Removed DiffStat `json:"removed"` - BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"` + MessageType string `json:"message_type"` // "statistics" + SourceSnapshot string `json:"source_snapshot"` + TargetSnapshot string `json:"target_snapshot"` + ChangedFiles int `json:"changed_files"` + Added DiffStat `json:"added"` + Removed DiffStat `json:"removed"` + BlobsBefore, BlobsAfter, BlobsCommon restic.AssociatedBlobSet `json:"-"` } // updateBlobs updates the blob counters in the stats struct. -func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, printError func(string, ...interface{})) { - for h := range blobs { +func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *DiffStat, printError func(string, ...interface{})) { + for h := range blobs.Keys() { switch h.Type { case restic.DataBlob: stats.DataBlobs++ @@ -177,7 +177,7 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, prin } } -func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error { +func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.AssociatedBlobSet, prefix string, id restic.ID) error { debug.Log("print %v tree %v", mode, id) tree, err := data.LoadTree(ctx, c.repo, id) if err != nil { @@ -208,7 +208,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b return ctx.Err() } -func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error { +func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSet, id restic.ID) error { debug.Log("print tree %v", id) tree, err := data.LoadTree(ctx, c.repo, id) if err != nil { @@ -442,9 +442,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [ MessageType: "statistics", SourceSnapshot: args[0], TargetSnapshot: args[1], - BlobsBefore: restic.NewBlobSet(), - BlobsAfter: restic.NewBlobSet(), - BlobsCommon: restic.NewBlobSet(), + BlobsBefore: repo.NewAssociatedBlobSet(), + BlobsAfter: repo.NewAssociatedBlobSet(), + BlobsCommon: repo.NewAssociatedBlobSet(), } stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree}) stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree}) From 46ebee948f1da83f100d3098a7fe03bad3bdfe2b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 19 Nov 2025 21:41:20 +0100 Subject: [PATCH 5/8] stats: use AssociatedBlobSet --- cmd/restic/cmd_stats.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 2b7a74b94..490806e2a 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -130,7 +130,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args stats := &statsContainer{ uniqueFiles: make(map[fileID]struct{}), fileBlobs: make(map[string]restic.IDSet), - blobs: restic.NewBlobSet(), + blobs: repo.NewAssociatedBlobSet(), SnapshotsCount: 0, } @@ -146,7 +146,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args if opts.countMode == countModeRawData { // the blob handles have been collected, but not yet counted - for blobHandle := range stats.blobs { + for blobHandle := range stats.blobs.Keys() { pbs := repo.LookupBlob(blobHandle.Type, blobHandle.ID) if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) @@ -350,7 +350,7 @@ type statsContainer struct { // blobs is used to count individual unique blobs, // independent of references to files - blobs restic.BlobSet + blobs restic.AssociatedBlobSet } // fileID is a 256-bit hash that distinguishes unique files. From 84dda4dc743eeff2a48ca6187cda4f6dc296d687 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 19 Nov 2025 21:41:44 +0100 Subject: [PATCH 6/8] check: use AssociatedBlobSet --- internal/checker/checker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 28c0f6fa5..5ef4a52b5 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -24,7 +24,7 @@ type Checker struct { *repository.Checker blobRefs struct { sync.Mutex - M restic.BlobSet + M restic.AssociatedBlobSet } trackUnused bool @@ -46,7 +46,7 @@ func New(repo checkerRepository, trackUnused bool) *Checker { trackUnused: trackUnused, } - c.blobRefs.M = restic.NewBlobSet() + c.blobRefs.M = c.repo.NewAssociatedBlobSet() return c } @@ -245,7 +245,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er c.blobRefs.Lock() defer c.blobRefs.Unlock() - debug.Log("checking %d blobs", len(c.blobRefs.M)) + debug.Log("checking %d blobs", c.blobRefs.M.Len()) ctx, cancel := context.WithCancel(ctx) defer cancel() From 7b59dd7cf4048aa1511ad70b58a5f277d5b6e4d7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 19 Nov 2025 21:56:16 +0100 Subject: [PATCH 7/8] add changelog --- changelog/unreleased/pull-5610 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/pull-5610 diff --git a/changelog/unreleased/pull-5610 b/changelog/unreleased/pull-5610 new file mode 100644 index 000000000..b6fd90821 --- /dev/null +++ b/changelog/unreleased/pull-5610 @@ -0,0 +1,7 @@ +Enhancement: reduce memory usage of check/copy/diff/stats commands + +We have optimized the memory usage of the `check`, `copy`, `diff` and +`stats` commands. These now require less memory when processing large +snapshots. + +https://github.com/restic/restic/pull/5610 From 134893bd35512dcc1c91b7c3f4c4043baf57091f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 22 Nov 2025 23:52:43 +0100 Subject: [PATCH 8/8] copy: use AssociatedBlobSet to keep track of processed trees --- cmd/restic/cmd_copy.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 823dcaf4e..fa81755d2 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -192,7 +192,7 @@ func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo res selectedSnapshots []*data.Snapshot, printer progress.Printer) error { // remember already processed trees across all snapshots - visitedTrees := restic.NewIDSet() + visitedTrees := srcRepo.NewAssociatedBlobSet() targetSize := uint64(dstRepo.PackSize()) * 100 minDuration := 1 * time.Minute @@ -242,13 +242,14 @@ func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo res } func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, - visitedTrees restic.IDSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaver) (uint64, error) { + visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaver) (uint64, error) { wg, wgCtx := errgroup.WithContext(ctx) treeStream := data.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool { - visited := visitedTrees.Has(treeID) - visitedTrees.Insert(treeID) + handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob} + visited := visitedTrees.Has(handle) + visitedTrees.Insert(handle) return visited }, nil)