amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-08-18 11:33:17 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5610 from MichaelEischer/associated-blob-set-everywhere

Browse Source 
check/copy/diff/stats: reduce memory usage
This commit is contained in:
Michael Eischer
2025-11-26 21:09:26 +01:00
committed by GitHub
parent 7e80536a9b 134893bd35
commit 4d2da63829
9 changed files with 176 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changelog/unreleased/pull-5610
+7
View File
@@ -0,0 +1,7 @@
Enhancement: reduce memory usage of check/copy/diff/stats commands
We have optimized the memory usage of the `check`, `copy`, `diff` and
`stats` commands. These now require less memory when processing large
snapshots.
https://github.com/restic/restic/pull/5610
cmd/restic/cmd_copy.go
+8 -7
View File
@@ -192,7 +192,7 @@ func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo res
selectedSnapshots []*data.Snapshot, printer progress.Printer) error {
// remember already processed trees across all snapshots
visitedTrees := restic.NewIDSet()
visitedTrees := srcRepo.NewAssociatedBlobSet()
targetSize := uint64(dstRepo.PackSize()) * 100
minDuration := 1 * time.Minute
@@ -242,17 +242,18 @@ func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo res
}
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
visitedTrees restic.IDSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaver) (uint64, error) {
visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaver) (uint64, error) {
wg, wgCtx := errgroup.WithContext(ctx)
treeStream := data.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
visited := visitedTrees.Has(treeID)
visitedTrees.Insert(treeID)
handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}
visited := visitedTrees.Has(handle)
visitedTrees.Insert(handle)
return visited
}, nil)
copyBlobs := restic.NewBlobSet()
copyBlobs := srcRepo.NewAssociatedBlobSet()
packList := restic.NewIDSet()
enqueue := func(h restic.BlobHandle) {
@@ -299,11 +300,11 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
}
// copyStats: print statistics for the blobs to be copied
func copyStats(srcRepo restic.Repository, copyBlobs restic.BlobSet, packList restic.IDSet, printer progress.Printer) uint64 {
func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, packList restic.IDSet, printer progress.Printer) uint64 {
// count and size
countBlobs := 0
sizeBlobs := uint64(0)
for blob := range copyBlobs {
for blob := range copyBlobs.Keys() {
for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) {
countBlobs++
sizeBlobs += uint64(blob.Length)
cmd/restic/cmd_diff.go
+15 -15
View File
@@ -124,7 +124,7 @@ func (s *DiffStat) Add(node *data.Node) {
}
// addBlobs adds the blobs of node to s.
func addBlobs(bs restic.BlobSet, node *data.Node) {
func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
if node == nil {
return
}
@@ -148,18 +148,18 @@ func addBlobs(bs restic.BlobSet, node *data.Node) {
}
type DiffStatsContainer struct {
MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.AssociatedBlobSet `json:"-"`
}
// updateBlobs updates the blob counters in the stats struct.
func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, printError func(string, ...interface{})) {
for h := range blobs {
func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *DiffStat, printError func(string, ...interface{})) {
for h := range blobs.Keys() {
switch h.Type {
case restic.DataBlob:
stats.DataBlobs++
@@ -177,7 +177,7 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat, prin
}
}
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.AssociatedBlobSet, prefix string, id restic.ID) error {
debug.Log("print %v tree %v", mode, id)
tree, err := data.LoadTree(ctx, c.repo, id)
if err != nil {
@@ -208,7 +208,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
return ctx.Err()
}
func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSet, id restic.ID) error {
debug.Log("print tree %v", id)
tree, err := data.LoadTree(ctx, c.repo, id)
if err != nil {
@@ -442,9 +442,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
MessageType: "statistics",
SourceSnapshot: args[0],
TargetSnapshot: args[1],
BlobsBefore: restic.NewBlobSet(),
BlobsAfter: restic.NewBlobSet(),
BlobsCommon: restic.NewBlobSet(),
BlobsBefore: repo.NewAssociatedBlobSet(),
BlobsAfter: repo.NewAssociatedBlobSet(),
BlobsCommon: repo.NewAssociatedBlobSet(),
}
stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
cmd/restic/cmd_stats.go
+3 -3
View File
@@ -130,7 +130,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
stats := &statsContainer{
uniqueFiles: make(map[fileID]struct{}),
fileBlobs: make(map[string]restic.IDSet),
blobs: restic.NewBlobSet(),
blobs: repo.NewAssociatedBlobSet(),
SnapshotsCount: 0,
}
@@ -146,7 +146,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
if opts.countMode == countModeRawData {
// the blob handles have been collected, but not yet counted
for blobHandle := range stats.blobs {
for blobHandle := range stats.blobs.Keys() {
pbs := repo.LookupBlob(blobHandle.Type, blobHandle.ID)
if len(pbs) == 0 {
return fmt.Errorf("blob %v not found", blobHandle)
@@ -350,7 +350,7 @@ type statsContainer struct {
// blobs is used to count individual unique blobs,
// independent of references to files
blobs restic.BlobSet
blobs restic.AssociatedBlobSet
}
// fileID is a 256-bit hash that distinguishes unique files.
internal/checker/checker.go
+3 -3
View File
@@ -24,7 +24,7 @@ type Checker struct {
*repository.Checker
blobRefs struct {
sync.Mutex
M restic.BlobSet
M restic.AssociatedBlobSet
}
trackUnused bool
@@ -46,7 +46,7 @@ func New(repo checkerRepository, trackUnused bool) *Checker {
trackUnused: trackUnused,
}
c.blobRefs.M = restic.NewBlobSet()
c.blobRefs.M = c.repo.NewAssociatedBlobSet()
return c
}
@@ -245,7 +245,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, er
c.blobRefs.Lock()
defer c.blobRefs.Unlock()
debug.Log("checking %d blobs", len(c.blobRefs.M))
debug.Log("checking %d blobs", c.blobRefs.M.Len())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
internal/repository/index/associated_data.go
+34
View File
@@ -108,6 +108,40 @@ func (a *AssociatedSet[T]) Delete(bh restic.BlobHandle) {
}
}
type haser interface {
Has(bh restic.BlobHandle) bool
}
// Intersect returns a new set containing the handles that are present in both sets.
func (a *AssociatedSet[T]) Intersect(other haser) *AssociatedSet[T] {
result := NewAssociatedSet[T](a.idx)
// Determining the smaller set already requires iterating over all keys
// and thus provides no performance benefit.
for bh := range a.Keys() {
if other.Has(bh) {
// preserve value receiver
val, _ := a.Get(bh)
result.Set(bh, val)
}
}
return result
}
// Sub returns a new set containing all handles that are present in a but not in
// other.
func (a *AssociatedSet[T]) Sub(other haser) *AssociatedSet[T] {
result := NewAssociatedSet[T](a.idx)
for bh := range a.Keys() {
if !other.Has(bh) {
val, _ := a.Get(bh)
result.Set(bh, val)
}
}
return result
}
func (a *AssociatedSet[T]) Len() int {
count := 0
for range a.All() {
internal/repository/index/associated_data_test.go
+78
View File
@@ -157,3 +157,81 @@ func TestAssociatedSetWithExtendedIndex(t *testing.T) {
test.Equals(t, list(bs), restic.BlobHandles(nil))
test.Equals(t, 0, len(bs.overflow))
}
func TestAssociatedSetIntersectAndSub(t *testing.T) {
mi := NewMasterIndex()
saver := &noopSaver{}
bh1, blob1 := makeFakePackedBlob()
bh2, blob2 := makeFakePackedBlob()
bh3, blob3 := makeFakePackedBlob()
bh4, blob4 := makeFakePackedBlob()
test.OK(t, mi.StorePack(context.TODO(), blob1.PackID, []restic.Blob{blob1.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob2.PackID, []restic.Blob{blob2.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob3.PackID, []restic.Blob{blob3.Blob}, saver))
test.OK(t, mi.StorePack(context.TODO(), blob4.PackID, []restic.Blob{blob4.Blob}, saver))
test.OK(t, mi.Flush(context.TODO(), saver))
t.Run("Intersect", func(t *testing.T) {
bs1, bs2 := NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
test.Equals(t, bs1.Intersect(bs2).Len(), 0)
bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
bs1.Set(bh1, 10)
bs2.Set(bh2, 20)
test.Equals(t, bs1.Intersect(bs2).Len(), 0)
bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
bs1.Set(bh3, 40)
bs2.Set(bh3, 50)
bs2.Set(bh4, 60)
result := bs1.Intersect(bs2)
test.Equals(t, result.Len(), 1)
val, _ := result.Get(bh3)
test.Equals(t, uint8(40), val)
bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
bs1.Set(bh3, 40)
bs1.Set(bh4, 70)
bs2.Set(bh3, 50)
bs2.Set(bh4, 60)
result = bs1.Intersect(bs2)
test.Equals(t, result.Len(), 2)
val, _ = result.Get(bh3)
test.Equals(t, uint8(40), val)
val, _ = result.Get(bh4)
test.Equals(t, uint8(70), val)
})
t.Run("Sub", func(t *testing.T) {
bs1, bs2 := NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
test.Equals(t, bs1.Sub(bs2).Len(), 0)
bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
bs1.Set(bh1, 10)
bs1.Set(bh2, 20)
bs2.Set(bh3, 30)
result := bs1.Sub(bs2)
test.Equals(t, result.Len(), 2)
val, _ := result.Get(bh1)
test.Equals(t, uint8(10), val)
val, _ = result.Get(bh2)
test.Equals(t, uint8(20), val)
bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
bs1.Set(bh1, 10)
bs1.Set(bh2, 20)
bs1.Set(bh3, 40)
bs2.Set(bh2, 50)
result = bs1.Sub(bs2)
test.Equals(t, result.Len(), 2)
test.Assert(t, result.Has(bh1) && result.Has(bh3) && !result.Has(bh2), "only bh1 and bh3 should be in result")
bs1, bs2 = NewAssociatedSet[uint8](mi), NewAssociatedSet[uint8](mi)
bs1.Set(bh1, 60)
bs2.Set(bh1, 70)
bs2.Set(bh2, 80)
test.Equals(t, bs1.Sub(bs2).Len(), 0)
})
}
internal/repository/repository.go
+16
View File
@@ -787,6 +787,22 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
return invalid, nil
}
func (r *Repository) NewAssociatedBlobSet() restic.AssociatedBlobSet {
return &associatedBlobSet{*index.NewAssociatedSet[struct{}](r.idx)}
}
// associatedBlobSet is a wrapper around index.AssociatedSet to implement the restic.AssociatedBlobSet interface.
type associatedBlobSet struct {
index.AssociatedSet[struct{}]
}
func (s *associatedBlobSet) Intersect(other restic.AssociatedBlobSet) restic.AssociatedBlobSet {
return &associatedBlobSet{*s.AssociatedSet.Intersect(other)}
}
func (s *associatedBlobSet) Sub(other restic.AssociatedBlobSet) restic.AssociatedBlobSet {
return &associatedBlobSet{*s.AssociatedSet.Sub(other)}
}
// prepareCache initializes the local cache. indexIDs is the list of IDs of
// index files still present in the repo.
func (r *Repository) prepareCache() error {
internal/restic/repository.go
+12
View File
@@ -2,6 +2,7 @@ package restic
import (
"context"
"iter"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto"
@@ -26,6 +27,7 @@ type Repository interface {
LookupBlob(t BlobType, id ID) []PackedBlob
LookupBlobSize(t BlobType, id ID) (size uint, exists bool)
NewAssociatedBlobSet() AssociatedBlobSet
// ListBlobs runs fn on all blobs known to the index. When the context is cancelled,
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
@@ -186,3 +188,13 @@ type FindBlobSet interface {
Has(bh BlobHandle) bool
Insert(bh BlobHandle)
}
type AssociatedBlobSet interface {
Has(bh BlobHandle) bool
Insert(bh BlobHandle)
Delete(bh BlobHandle)
Len() int
Keys() iter.Seq[BlobHandle]
Intersect(other AssociatedBlobSet) AssociatedBlobSet
Sub(other AssociatedBlobSet) AssociatedBlobSet
}