data: replace Tree with TreeNodeIterator

The TreeNodeIterator decodes nodes while iterating over a tree blob.
This should reduce peak memory usage as now only the serialized tree
blob and a single node have to alive at the same time. Using the
iterator has implications for the error handling however. Now it is
necessary that all loops that iterate through a tree check for errors
before using the node returned by the iterator.

The other change is that it is no longer possible to iterate over a tree
multiple times. Instead it must be loaded a second time. This only
affects the tree rewriting code.
This commit is contained in:
Michael Eischer committed 2026-01-31 20:03:38 +01:00
1 parent 1e183509d4
commit 350f29d921
23 files changed
+394 -233

No files matched your search

+15 -7
View File
@@ -7,6 +7,7 @@ import (
"errors"
"os"
"path/filepath"
"slices"
"sync"
"syscall"
@@ -65,13 +66,13 @@ func unwrapCtxCanceled(err error) error {
// replaceSpecialNodes replaces nodes with name "." and "/" by their contents.
// Otherwise, the node is returned.
func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data.Node) ([]*data.Node, error) {
func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data.Node) (data.TreeNodeIterator, error) {
if node.Type != data.NodeTypeDir || node.Subtree == nil {
return []*data.Node{node}, nil
return slices.Values([]data.NodeOrError{{Node: node}}), nil
}
if node.Name != "." && node.Name != "/" {
return []*data.Node{node}, nil
return slices.Values([]data.NodeOrError{{Node: node}}), nil
}
tree, err := data.LoadTree(ctx, repo, *node.Subtree)
@@ -79,7 +80,7 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data
return nil, unwrapCtxCanceled(err)
}
return tree.Nodes, nil
return tree, nil
}
func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *data.Snapshot) (*dir, error) {
@@ -115,18 +116,25 @@ func (d *dir) open(ctx context.Context) error {
return unwrapCtxCanceled(err)
}
items := make(map[string]*data.Node)
for _, n := range tree.Nodes {
for item := range tree {
if item.Error != nil {
return unwrapCtxCanceled(item.Error)
}
if ctx.Err() != nil {
return ctx.Err()
}
n := item.Node
nodes, err := replaceSpecialNodes(ctx, d.root.repo, n)
if err != nil {
debug.Log(" replaceSpecialNodes(%v) failed: %v", n, err)
return err
}
for _, node := range nodes {
items[cleanupNodeName(node.Name)] = node
for item := range nodes {
if item.Error != nil {
return unwrapCtxCanceled(item.Error)
}
items[cleanupNodeName(item.Node.Name)] = item.Node
}
}
d.items = items
+4 -3
View File
@@ -59,7 +59,7 @@ func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *data.Sna
return sn
}
func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *data.Tree {
func loadTree(t testing.TB, repo restic.Loader, id restic.ID) data.TreeNodeIterator {
tree, err := data.LoadTree(context.TODO(), repo, id)
rtest.OK(t, err)
return tree
@@ -79,8 +79,9 @@ func TestFuseFile(t *testing.T) {
tree := loadTree(t, repo, *sn.Tree)
var content restic.IDs
for _, node := range tree.Nodes {
content = append(content, node.Content...)
for item := range tree {
rtest.OK(t, item.Error)
content = append(content, item.Node.Content...)
}
t.Logf("tree loaded, content: %v", content)