mirror of
https://github.com/restic/restic.git
synced 2026-11-21 21:12:42 +00:00
data: replace Tree with TreeNodeIterator
The TreeNodeIterator decodes nodes while iterating over a tree blob. This should reduce peak memory usage as now only the serialized tree blob and a single node have to alive at the same time. Using the iterator has implications for the error handling however. Now it is necessary that all loops that iterate through a tree check for errors before using the node returned by the iterator. The other change is that it is no longer possible to iterate over a tree multiple times. Instead it must be loaded a second time. This only affects the tree rewriting code.
This commit is contained in:
1 parent
1e183509d4
commit
350f29d921
23 files changed
+394
-233
No files matched your search
+15
-7
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
@@ -65,13 +66,13 @@ func unwrapCtxCanceled(err error) error {
|
||||
|
||||
// replaceSpecialNodes replaces nodes with name "." and "/" by their contents.
|
||||
// Otherwise, the node is returned.
|
||||
func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data.Node) ([]*data.Node, error) {
|
||||
func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data.Node) (data.TreeNodeIterator, error) {
|
||||
if node.Type != data.NodeTypeDir || node.Subtree == nil {
|
||||
return []*data.Node{node}, nil
|
||||
return slices.Values([]data.NodeOrError{{Node: node}}), nil
|
||||
}
|
||||
|
||||
if node.Name != "." && node.Name != "/" {
|
||||
return []*data.Node{node}, nil
|
||||
return slices.Values([]data.NodeOrError{{Node: node}}), nil
|
||||
}
|
||||
|
||||
tree, err := data.LoadTree(ctx, repo, *node.Subtree)
|
||||
@@ -79,7 +80,7 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *data
|
||||
return nil, unwrapCtxCanceled(err)
|
||||
}
|
||||
|
||||
return tree.Nodes, nil
|
||||
return tree, nil
|
||||
}
|
||||
|
||||
func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *data.Snapshot) (*dir, error) {
|
||||
@@ -115,18 +116,25 @@ func (d *dir) open(ctx context.Context) error {
|
||||
return unwrapCtxCanceled(err)
|
||||
}
|
||||
items := make(map[string]*data.Node)
|
||||
for _, n := range tree.Nodes {
|
||||
for item := range tree {
|
||||
if item.Error != nil {
|
||||
return unwrapCtxCanceled(item.Error)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
n := item.Node
|
||||
|
||||
nodes, err := replaceSpecialNodes(ctx, d.root.repo, n)
|
||||
if err != nil {
|
||||
debug.Log(" replaceSpecialNodes(%v) failed: %v", n, err)
|
||||
return err
|
||||
}
|
||||
for _, node := range nodes {
|
||||
items[cleanupNodeName(node.Name)] = node
|
||||
for item := range nodes {
|
||||
if item.Error != nil {
|
||||
return unwrapCtxCanceled(item.Error)
|
||||
}
|
||||
items[cleanupNodeName(item.Node.Name)] = item.Node
|
||||
}
|
||||
}
|
||||
d.items = items
|
||||
|
||||
@@ -59,7 +59,7 @@ func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *data.Sna
|
||||
return sn
|
||||
}
|
||||
|
||||
func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *data.Tree {
|
||||
func loadTree(t testing.TB, repo restic.Loader, id restic.ID) data.TreeNodeIterator {
|
||||
tree, err := data.LoadTree(context.TODO(), repo, id)
|
||||
rtest.OK(t, err)
|
||||
return tree
|
||||
@@ -79,8 +79,9 @@ func TestFuseFile(t *testing.T) {
|
||||
tree := loadTree(t, repo, *sn.Tree)
|
||||
|
||||
var content restic.IDs
|
||||
for _, node := range tree.Nodes {
|
||||
content = append(content, node.Content...)
|
||||
for item := range tree {
|
||||
rtest.OK(t, item.Error)
|
||||
content = append(content, item.Node.Content...)
|
||||
}
|
||||
t.Logf("tree loaded, content: %v", content)
|
||||
|
||||
|
||||
Reference in new issue
Block a user