mirror of
https://github.com/restic/restic.git
synced 2026-11-21 13:02:42 +00:00
data: read empty trees from older repaired snapshots
Older repair snapshots versions serialized missing subtrees as nodes:null. The streaming decoder rejects these trees, preventing prune and other commands from traversing snapshots that previous releases could read. Accept null as an empty node list. Add decoder coverage and a regression that prunes, checks and restores a snapshot with a legacy empty subtree.
This commit is contained in:
1 parent
9da3b87b07
commit
d714e22605
4 files changed
+116
-13
No files matched your search
@@ -0,0 +1,8 @@
|
||||
Bugfix: Read empty directories from older repaired snapshots
|
||||
|
||||
Since restic 0.19.0, snapshots containing empty directories written by older
|
||||
versions of `repair snapshots` could fail with `error decoding tree: expected
|
||||
[, got <nil>`. Restic now accepts these empty directory trees again, allowing
|
||||
`prune`, `check` and `restore` to process the affected snapshots.
|
||||
|
||||
https://github.com/restic/restic/issues/22092
|
||||
@@ -3,12 +3,16 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/data"
|
||||
"github.com/restic/restic/internal/global"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
@@ -123,6 +127,53 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
|
||||
|
||||
var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
|
||||
|
||||
func TestPruneWithLegacyEmptyTree(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
testRunInit(t, env.gopts)
|
||||
|
||||
ctx := context.Background()
|
||||
var repo *repository.Repository
|
||||
rtest.OK(t, withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
|
||||
var err error
|
||||
repo, err = global.OpenRepository(ctx, gopts, restic.NewNoopPrinter())
|
||||
return err
|
||||
}))
|
||||
defer func() { rtest.OK(t, repo.Close()) }()
|
||||
contents := []byte("file next to a repaired empty directory")
|
||||
mtime := time.Unix(1700000000, 0)
|
||||
var treeID restic.ID
|
||||
rtest.OK(t, repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
// repair snapshots in restic 0.18.1 saved missing subtrees as {"nodes":null}.
|
||||
emptyID, _, _, err := uploader.SaveBlob(ctx, restic.TreeBlob, []byte("{\"nodes\":null}\n"), restic.ID{}, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileID, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, contents, restic.ID{}, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{
|
||||
{Name: "empty", Type: data.NodeTypeDir, Mode: 0755, ModTime: mtime, AccessTime: mtime, Subtree: &emptyID},
|
||||
{Name: "keep.txt", Type: data.NodeTypeFile, Mode: 0644, ModTime: mtime, AccessTime: mtime, Size: uint64(len(contents)), Content: restic.IDs{fileID}},
|
||||
})
|
||||
return nil
|
||||
}))
|
||||
_, err := data.SaveSnapshot(ctx, repo, &data.Snapshot{Tree: &treeID, Time: mtime, Paths: []string{"/test"}})
|
||||
rtest.OK(t, err)
|
||||
|
||||
testRunPrune(t, env.gopts, pruneDefaultOptions)
|
||||
testRunCheck(t, env.gopts)
|
||||
target := t.TempDir()
|
||||
testRunRestore(t, env.gopts, target, "latest")
|
||||
got, err := os.ReadFile(filepath.Join(target, "keep.txt"))
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, contents, got)
|
||||
entries, err := os.ReadDir(filepath.Join(target, "empty"))
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, 0, len(entries))
|
||||
}
|
||||
|
||||
func TestPruneWithDamagedRepository(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
+21
-13
@@ -25,8 +25,9 @@ import (
|
||||
var ErrTreeNotOrdered = errors.New("nodes are not ordered or duplicate")
|
||||
|
||||
type treeIterator struct {
|
||||
dec json.Decoder
|
||||
started bool
|
||||
dec json.Decoder
|
||||
started bool
|
||||
nodesNull bool
|
||||
}
|
||||
|
||||
type NodeOrError struct {
|
||||
@@ -87,10 +88,15 @@ func (t *treeIterator) init() error {
|
||||
return errors.Errorf("error decoding tree: expected string key, got %v", token)
|
||||
}
|
||||
if key == "nodes" {
|
||||
// Found "nodes", proceed to read the array
|
||||
if err := t.assertToken(json.Delim('[')); err != nil {
|
||||
token, err := t.dec.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if token != nil && token != json.Delim('[') {
|
||||
return errors.Errorf("error decoding tree: expected [, got %v", token)
|
||||
}
|
||||
// A null node list encodes an empty tree in older repaired snapshots.
|
||||
t.nodesNull = token == nil
|
||||
return nil
|
||||
}
|
||||
// Unknown key, decode its value into RawMessage and discard it
|
||||
@@ -102,17 +108,19 @@ func (t *treeIterator) init() error {
|
||||
}
|
||||
|
||||
func (t *treeIterator) next() (*Node, error) {
|
||||
if t.dec.More() {
|
||||
var n Node
|
||||
err := t.dec.Decode(&n)
|
||||
if err != nil {
|
||||
if !t.nodesNull {
|
||||
if t.dec.More() {
|
||||
var n Node
|
||||
err := t.dec.Decode(&n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &n, nil
|
||||
}
|
||||
|
||||
if err := t.assertToken(json.Delim(']')); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &n, nil
|
||||
}
|
||||
|
||||
if err := t.assertToken(json.Delim(']')); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Skip unknown keys after the array until we find the closing brace
|
||||
for {
|
||||
|
||||
@@ -295,6 +295,42 @@ func TestTreeIteratorUnknownKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTreeIteratorNullNodes(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
input string
|
||||
wantErr bool
|
||||
}{
|
||||
{"null", `{"nodes":null}`, false},
|
||||
{"unknown key before", `{"extra":{"nested":[1,2]},"nodes":null}`, false},
|
||||
{"unknown key after", `{"nodes":null,"extra":{"nested":[1,2]}}`, false},
|
||||
{"unknown keys around", `{"before":true,"nodes":null,"after":[1,2]}`, false},
|
||||
{"boolean", `{"nodes":false}`, true},
|
||||
{"number", `{"nodes":0}`, true},
|
||||
{"string", `{"nodes":""}`, true},
|
||||
{"object", `{"nodes":{}}`, true},
|
||||
{"invalid trailing value", `{"nodes":null,"extra":]}`, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
it, err := data.NewTreeNodeIterator(strings.NewReader(tc.input))
|
||||
items := 0
|
||||
if err == nil {
|
||||
for item := range it {
|
||||
items++
|
||||
err = item.Error
|
||||
rtest.Assert(t, item.Node == nil, "unexpected node: %v", item.Node)
|
||||
}
|
||||
}
|
||||
if tc.wantErr {
|
||||
rtest.Assert(t, err != nil, "expected invalid tree to fail")
|
||||
} else {
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, 0, items)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLoadTree(t *testing.B) {
|
||||
repository.BenchmarkAllVersions(t, benchmarkLoadTree)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user