mirror of
https://github.com/restic/restic.git
synced 2026-11-21 21:12:42 +00:00
data: replace Tree with TreeNodeIterator
The TreeNodeIterator decodes nodes while iterating over a tree blob. This should reduce peak memory usage as now only the serialized tree blob and a single node have to alive at the same time. Using the iterator has implications for the error handling however. Now it is necessary that all loops that iterate through a tree check for errors before using the node returned by the iterator. The other change is that it is no longer possible to iterate over a tree multiple times. Instead it must be loaded a second time. This only affects the tree rewriting code.
This commit is contained in:
1 parent
1e183509d4
commit
350f29d921
23 files changed
+394
-233
No files matched your search
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
type NodeRewriteFunc func(node *data.Node, path string) *data.Node
|
||||
type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (*data.Tree, error)
|
||||
type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (data.TreeNodeIterator, error)
|
||||
type QueryRewrittenSizeFunc func() SnapshotSize
|
||||
|
||||
type SnapshotSize struct {
|
||||
@@ -52,7 +52,7 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter {
|
||||
}
|
||||
if rw.opts.RewriteFailedTree == nil {
|
||||
// fail with error by default
|
||||
rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (*data.Tree, error) {
|
||||
rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (data.TreeNodeIterator, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -117,15 +117,26 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, loader restic.BlobLoader
|
||||
if nodeID != testID {
|
||||
return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath)
|
||||
}
|
||||
|
||||
// reload the tree to get a new iterator
|
||||
curTree, err = data.LoadTree(ctx, loader, nodeID)
|
||||
if err != nil {
|
||||
// shouldn't fail as the first load was successful
|
||||
return restic.ID{}, fmt.Errorf("failed to reload tree %v: %w", nodeID, err)
|
||||
}
|
||||
}
|
||||
|
||||
debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str())
|
||||
|
||||
tb := data.NewTreeWriter(saver)
|
||||
for _, node := range curTree.Nodes {
|
||||
for item := range curTree {
|
||||
if ctx.Err() != nil {
|
||||
return restic.ID{}, ctx.Err()
|
||||
}
|
||||
if item.Error != nil {
|
||||
return restic.ID{}, err
|
||||
}
|
||||
node := item.Node
|
||||
|
||||
path := path.Join(nodepath, node.Name)
|
||||
node = t.opts.RewriteNode(node, path)
|
||||
|
||||
@@ -2,6 +2,7 @@ package walker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
@@ -405,16 +406,15 @@ func TestRewriterTreeLoadError(t *testing.T) {
|
||||
t.Fatal("missing error on unloadable tree")
|
||||
}
|
||||
|
||||
replacementTree := &data.Tree{Nodes: []*data.Node{{Name: "replacement", Type: data.NodeTypeFile, Size: 42}}}
|
||||
replacementID, err := data.SaveTree(ctx, tm, replacementTree)
|
||||
test.OK(t, err)
|
||||
replacementNode := &data.Node{Name: "replacement", Type: data.NodeTypeFile, Size: 42}
|
||||
replacementID := data.TestSaveNodes(t, ctx, tm, []*data.Node{replacementNode})
|
||||
|
||||
rewriter = NewTreeRewriter(RewriteOpts{
|
||||
RewriteFailedTree: func(nodeID restic.ID, path string, err error) (*data.Tree, error) {
|
||||
RewriteFailedTree: func(nodeID restic.ID, path string, err error) (data.TreeNodeIterator, error) {
|
||||
if nodeID != id || path != "/" {
|
||||
t.Fail()
|
||||
}
|
||||
return replacementTree, nil
|
||||
return slices.Values([]data.NodeOrError{{Node: replacementNode}}), nil
|
||||
},
|
||||
})
|
||||
newRoot, err := rewriter.RewriteTree(ctx, tm, tm, "/", id)
|
||||
|
||||
@@ -3,7 +3,6 @@ package walker
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"sort"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
@@ -52,15 +51,15 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor W
|
||||
// walk recursively traverses the tree, ignoring subtrees when the ID of the
|
||||
// subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID
|
||||
// will be added to ignoreTrees by walk.
|
||||
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *data.Tree, visitor WalkVisitor) (err error) {
|
||||
sort.Slice(tree.Nodes, func(i, j int) bool {
|
||||
return tree.Nodes[i].Name < tree.Nodes[j].Name
|
||||
})
|
||||
|
||||
for _, node := range tree.Nodes {
|
||||
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree data.TreeNodeIterator, visitor WalkVisitor) (err error) {
|
||||
for item := range tree {
|
||||
if item.Error != nil {
|
||||
return item.Error
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
node := item.Node
|
||||
|
||||
p := path.Join(prefix, node.Name)
|
||||
|
||||
|
||||
Reference in new issue
Block a user