data: replace Tree with TreeNodeIterator

The TreeNodeIterator decodes nodes while iterating over a tree blob.
This should reduce peak memory usage as now only the serialized tree
blob and a single node have to alive at the same time. Using the
iterator has implications for the error handling however. Now it is
necessary that all loops that iterate through a tree check for errors
before using the node returned by the iterator.

The other change is that it is no longer possible to iterate over a tree
multiple times. Instead it must be loaded a second time. This only
affects the tree rewriting code.
This commit is contained in:
Michael Eischer committed 2026-01-31 20:03:38 +01:00
1 parent 1e183509d4
commit 350f29d921
23 files changed
+394 -233

No files matched your search

+14 -3
View File
@@ -11,7 +11,7 @@ import (
)
type NodeRewriteFunc func(node *data.Node, path string) *data.Node
type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (*data.Tree, error)
type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (data.TreeNodeIterator, error)
type QueryRewrittenSizeFunc func() SnapshotSize
type SnapshotSize struct {
@@ -52,7 +52,7 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter {
}
if rw.opts.RewriteFailedTree == nil {
// fail with error by default
rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (*data.Tree, error) {
rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (data.TreeNodeIterator, error) {
return nil, err
}
}
@@ -117,15 +117,26 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, loader restic.BlobLoader
if nodeID != testID {
return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath)
}
// reload the tree to get a new iterator
curTree, err = data.LoadTree(ctx, loader, nodeID)
if err != nil {
// shouldn't fail as the first load was successful
return restic.ID{}, fmt.Errorf("failed to reload tree %v: %w", nodeID, err)
}
}
debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str())
tb := data.NewTreeWriter(saver)
for _, node := range curTree.Nodes {
for item := range curTree {
if ctx.Err() != nil {
return restic.ID{}, ctx.Err()
}
if item.Error != nil {
return restic.ID{}, err
}
node := item.Node
path := path.Join(nodepath, node.Name)
node = t.opts.RewriteNode(node, path)
+5 -5
View File
@@ -2,6 +2,7 @@ package walker
import (
"context"
"slices"
"testing"
"github.com/pkg/errors"
@@ -405,16 +406,15 @@ func TestRewriterTreeLoadError(t *testing.T) {
t.Fatal("missing error on unloadable tree")
}
replacementTree := &data.Tree{Nodes: []*data.Node{{Name: "replacement", Type: data.NodeTypeFile, Size: 42}}}
replacementID, err := data.SaveTree(ctx, tm, replacementTree)
test.OK(t, err)
replacementNode := &data.Node{Name: "replacement", Type: data.NodeTypeFile, Size: 42}
replacementID := data.TestSaveNodes(t, ctx, tm, []*data.Node{replacementNode})
rewriter = NewTreeRewriter(RewriteOpts{
RewriteFailedTree: func(nodeID restic.ID, path string, err error) (*data.Tree, error) {
RewriteFailedTree: func(nodeID restic.ID, path string, err error) (data.TreeNodeIterator, error) {
if nodeID != id || path != "/" {
t.Fail()
}
return replacementTree, nil
return slices.Values([]data.NodeOrError{{Node: replacementNode}}), nil
},
})
newRoot, err := rewriter.RewriteTree(ctx, tm, tm, "/", id)
+6 -7
View File
@@ -3,7 +3,6 @@ package walker
import (
"context"
"path"
"sort"
"github.com/pkg/errors"
@@ -52,15 +51,15 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor W
// walk recursively traverses the tree, ignoring subtrees when the ID of the
// subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID
// will be added to ignoreTrees by walk.
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *data.Tree, visitor WalkVisitor) (err error) {
sort.Slice(tree.Nodes, func(i, j int) bool {
return tree.Nodes[i].Name < tree.Nodes[j].Name
})
for _, node := range tree.Nodes {
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree data.TreeNodeIterator, visitor WalkVisitor) (err error) {
for item := range tree {
if item.Error != nil {
return item.Error
}
if ctx.Err() != nil {
return ctx.Err()
}
node := item.Node
p := path.Join(prefix, node.Name)