mirror of
https://github.com/restic/restic.git
synced 2026-09-14 16:27:59 +00:00
Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6aa3a516ce | ||
|
|
b1da29738c | ||
|
|
09a1eb6714 | ||
|
|
c2b4d6a959 | ||
|
|
437945bf90 | ||
|
|
fc422d4145 | ||
|
|
f058670219 | ||
|
|
df8454770b | ||
|
|
c9185dd19c | ||
|
|
32be2e559b | ||
|
|
add4fa1efb | ||
|
|
c7c3414641 | ||
|
|
8ef295e2f5 | ||
|
|
bc714d1864 | ||
|
|
debf95f58d | ||
|
|
27c44c11aa | ||
|
|
93f695a803 | ||
|
|
3e21d1fea6 | ||
|
|
8ee674da55 | ||
|
|
c21f0fabc6 | ||
|
|
6aaa2eace4 | ||
|
|
f2d614db56 | ||
|
|
ed655b476a | ||
|
|
80f694b32e | ||
|
|
eaa835becd | ||
|
|
7aa82d68b4 | ||
|
|
a3fa3eb182 | ||
|
|
cb24c4f566 | ||
|
|
d6fd4ae26f | ||
|
|
4134bdfac2 | ||
|
|
c08e7fd956 | ||
|
|
d07bcd50da | ||
|
|
e6b9118f51 | ||
|
|
82a665384b | ||
|
|
1e249b0da1 | ||
|
|
5967f9a894 | ||
|
|
1733dd56b9 | ||
|
|
ed39b60e7d |
+1
-7
@@ -32,6 +32,7 @@ linters:
|
||||
backend-imports:
|
||||
files:
|
||||
- "**/internal/backend/**"
|
||||
- "!**/internal/backend/cache/**"
|
||||
- "!**/internal/backend/test/**"
|
||||
- "!**/*_test.go"
|
||||
deny:
|
||||
@@ -39,13 +40,6 @@ linters:
|
||||
desc: "internal/restic should not be imported to keep the architectural layers intact"
|
||||
- pkg: "github.com/restic/restic/internal/repository"
|
||||
desc: "internal/repository should not be imported to keep the architectural layers intact"
|
||||
repository-internals:
|
||||
files:
|
||||
- "**"
|
||||
- "!**/internal/repository/**"
|
||||
deny:
|
||||
- pkg: "github.com/restic/restic/internal/repository/"
|
||||
desc: "packages below internal/repository should not be imported to not depend on repository internals"
|
||||
importas:
|
||||
alias:
|
||||
- pkg: github.com/restic/restic/internal/test
|
||||
|
||||
+111
@@ -1,5 +1,6 @@
|
||||
# Table of Contents
|
||||
|
||||
* [Changelog for 0.19.1](#changelog-for-restic-0191-2026-07-05)
|
||||
* [Changelog for 0.19.0](#changelog-for-restic-0190-2026-06-09)
|
||||
* [Changelog for 0.18.1](#changelog-for-restic-0181-2025-09-21)
|
||||
* [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
|
||||
@@ -41,6 +42,116 @@
|
||||
* [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
|
||||
|
||||
|
||||
# Changelog for restic 0.19.1 (2026-07-05)
|
||||
The following sections list the changes in restic 0.19.1 relevant to
|
||||
restic users. The changes are ordered by importance.
|
||||
|
||||
## Summary
|
||||
|
||||
* Fix #5234: Prevent mounting over the repository directory
|
||||
* Fix #5667: Skip inaccessible `backup` source paths
|
||||
* Fix #5722: Update `mount` latest symlink after snapshot reload
|
||||
* Fix #21866: Hide `stats` progress bar in JSON mode
|
||||
* Fix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
|
||||
* Fix #21876: Show timezone location in `snapshots` output
|
||||
* Fix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
|
||||
* Fix #21895: Remove read-only files via the SFTP backend on Windows servers
|
||||
* Fix #21899: Make `backup` respect excludes for duplicate directory entries
|
||||
|
||||
## Details
|
||||
|
||||
* Bugfix #5234: Prevent mounting over the repository directory
|
||||
|
||||
Using a local repository directory as the `mount` target — or a path that
|
||||
contains it, or that it contains — caused the FUSE server to read its own
|
||||
backend files through the new mount, deadlocking the kernel and requiring a long
|
||||
reboot to recover.
|
||||
|
||||
Restic now resolves both paths and refuses any such overlap with a clear error
|
||||
before mounting.
|
||||
|
||||
https://github.com/restic/restic/issues/5234
|
||||
https://github.com/restic/restic/pull/5348
|
||||
|
||||
* Bugfix #5667: Skip inaccessible `backup` source paths
|
||||
|
||||
The `backup` command only skipped source paths that did not exist. A path that
|
||||
could not be accessed for another reason, such as a malformed path on Windows,
|
||||
was kept and produced an empty snapshot. Restic now skips any such path and
|
||||
aborts if none remain.
|
||||
|
||||
https://github.com/restic/restic/issues/5667
|
||||
https://github.com/restic/restic/pull/21852
|
||||
|
||||
* Bugfix #5722: Update `mount` latest symlink after snapshot reload
|
||||
|
||||
When `restic mount` was kept running while new snapshots were created, the new
|
||||
snapshots appeared in the mountpoint, but the `latest` symlink could still point
|
||||
to the previously latest snapshot. Restic now invalidates the cached snapshot
|
||||
directory entries after a snapshot reload so that `latest` points to the newest
|
||||
snapshot.
|
||||
|
||||
https://github.com/restic/restic/issues/5722
|
||||
https://github.com/restic/restic/pull/21873
|
||||
|
||||
* Bugfix #21866: Hide `stats` progress bar in JSON mode
|
||||
|
||||
Since restic 0.19.0, the `stats` command shows a progress bar. This progress bar
|
||||
was unintentionally displayed also when using the `--json` option, mixing
|
||||
regular text output with JSON. This is now fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/21866
|
||||
https://github.com/restic/restic/pull/21871
|
||||
|
||||
* Bugfix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
|
||||
|
||||
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>` to
|
||||
no longer group snapshots by host and paths by default.
|
||||
|
||||
The `snapshots --latest <n>` command now again uses the old behavior of grouping
|
||||
by host and paths when `--group-by` is not specified. However, when specifying
|
||||
`--group-by` the output is still grouped as requested, as in restic 0.19.0.
|
||||
|
||||
https://github.com/restic/restic/issues/21869
|
||||
https://github.com/restic/restic/pull/21875
|
||||
|
||||
* Bugfix #21876: Show timezone location in `snapshots` output
|
||||
|
||||
With restic 0.19.0, the `snapshots` command printed the current timezone when
|
||||
listing snapshots. However, that timezone label might change during the year,
|
||||
for example with daylight saving time. Restic now prints a more consistent and
|
||||
shorter version of the text.
|
||||
|
||||
https://github.com/restic/restic/pull/21876
|
||||
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
|
||||
|
||||
* Bugfix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
|
||||
|
||||
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
|
||||
the repository. If restic was unable to stat the mountpoint, this would result
|
||||
in a crash. This has now been fixed to correctly return an error instead.
|
||||
|
||||
https://github.com/restic/restic/pull/21879
|
||||
|
||||
* Bugfix #21895: Remove read-only files via the SFTP backend on Windows servers
|
||||
|
||||
Since restic 0.19.0, repository files on the SFTP backend are marked read-only
|
||||
after save. On Windows SFTP servers, removing them failed with a permission
|
||||
error. The SFTP backend now clears the read-only flag before removing the file.
|
||||
|
||||
https://github.com/restic/restic/issues/21895
|
||||
https://github.com/restic/restic/pull/21897
|
||||
|
||||
* Bugfix #21899: Make `backup` respect excludes for duplicate directory entries
|
||||
|
||||
Since restic 0.19.0, backing up a directory with duplicate directory entries
|
||||
always produced "Warning: at least one source file could not be read", even when
|
||||
those files were excluded. This has now been fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/21899
|
||||
https://github.com/restic/restic/pull/21900
|
||||
|
||||
|
||||
# Changelog for restic 0.19.0 (2026-06-09)
|
||||
The following sections list the changes in restic 0.19.0 relevant to
|
||||
restic users. The changes are ordered by importance.
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
Bugfix: Hide `stats` progress bar in JSON mode
|
||||
|
||||
Since restic 0.19.0, the `stats` command shows a progress bar. This
|
||||
progress bar was unintentionally displayed also when using the `--json`
|
||||
option, mixing regular text output with JSON. This is now fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/21866
|
||||
https://github.com/restic/restic/pull/21871
|
||||
@@ -0,0 +1,12 @@
|
||||
Bugfix: Restore old behavior of `snapshots --latest <n>` without `--group-by`
|
||||
|
||||
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>`
|
||||
to no longer group snapshots by host and paths by default.
|
||||
|
||||
The `snapshots --latest <n>` command now again uses the old behavior of
|
||||
grouping by host and paths when `--group-by` is not specified. However, when
|
||||
specifying `--group-by` the output is still grouped as requested, as in restic
|
||||
0.19.0.
|
||||
|
||||
https://github.com/restic/restic/issues/21869
|
||||
https://github.com/restic/restic/pull/21875
|
||||
@@ -0,0 +1,9 @@
|
||||
Bugfix: Remove read-only files via the SFTP backend on Windows servers
|
||||
|
||||
Since restic 0.19.0, repository files on the SFTP backend are marked
|
||||
read-only after save. On Windows SFTP servers, removing them failed
|
||||
with a permission error. The SFTP backend now clears the read-only flag
|
||||
before removing the file.
|
||||
|
||||
https://github.com/restic/restic/issues/21895
|
||||
https://github.com/restic/restic/pull/21897
|
||||
@@ -0,0 +1,8 @@
|
||||
Bugfix: Make `backup` respect excludes for duplicate directory entries
|
||||
|
||||
Since restic 0.19.0, backing up a directory with duplicate directory
|
||||
entries always produced "Warning: at least one source file could not be
|
||||
read", even when those files were excluded. This has now been fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/21899
|
||||
https://github.com/restic/restic/pull/21900
|
||||
@@ -0,0 +1,12 @@
|
||||
Bugfix: Prevent mounting over the repository directory
|
||||
|
||||
Using a local repository directory as the `mount` target — or a path
|
||||
that contains it, or that it contains — caused the FUSE server to
|
||||
read its own backend files through the new mount, deadlocking the
|
||||
kernel and requiring a long reboot to recover.
|
||||
|
||||
Restic now resolves both paths and refuses any such overlap with a
|
||||
clear error before mounting.
|
||||
|
||||
https://github.com/restic/restic/issues/5234
|
||||
https://github.com/restic/restic/pull/5348
|
||||
@@ -0,0 +1,9 @@
|
||||
Bugfix: Skip inaccessible `backup` source paths
|
||||
|
||||
The `backup` command only skipped source paths that did not exist. A path that
|
||||
could not be accessed for another reason, such as a malformed path on Windows,
|
||||
was kept and produced an empty snapshot. Restic now skips any such path and
|
||||
aborts if none remain.
|
||||
|
||||
https://github.com/restic/restic/issues/5667
|
||||
https://github.com/restic/restic/pull/21852
|
||||
@@ -0,0 +1,10 @@
|
||||
Bugfix: Update `mount` latest symlink after snapshot reload
|
||||
|
||||
When `restic mount` was kept running while new snapshots were
|
||||
created, the new snapshots appeared in the mountpoint, but the `latest`
|
||||
symlink could still point to the previously latest snapshot. Restic now
|
||||
invalidates the cached snapshot directory entries after a snapshot reload so
|
||||
that `latest` points to the newest snapshot.
|
||||
|
||||
https://github.com/restic/restic/issues/5722
|
||||
https://github.com/restic/restic/pull/21873
|
||||
@@ -0,0 +1,9 @@
|
||||
Bugfix: Show timezone location in `snapshots` output
|
||||
|
||||
With restic 0.19.0, the `snapshots` command printed the current timezone when
|
||||
listing snapshots. However, that timezone label might change during the year,
|
||||
for example with daylight saving time. Restic now prints a more consistent and
|
||||
shorter version of the text.
|
||||
|
||||
https://github.com/restic/restic/pull/21876
|
||||
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
|
||||
@@ -0,0 +1,7 @@
|
||||
Bugfix: Prevent crash in mountpoint validation if mountpoint is inaccessible
|
||||
|
||||
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
|
||||
the repository. If restic was unable to stat the mountpoint, this would result
|
||||
in a crash. This has now been fixed to correctly return an error instead.
|
||||
|
||||
https://github.com/restic/restic/pull/21879
|
||||
@@ -171,13 +171,17 @@ var ErrInvalidSourceData = errors.New("at least one source file could not be rea
|
||||
// ErrNoSourceData is used to report that no source data was found
|
||||
var ErrNoSourceData = errors.Fatal("all source directories/files do not exist")
|
||||
|
||||
// filterExisting returns a slice of all existing items, or an error if no
|
||||
// items exist at all.
|
||||
// filterExisting returns the items that exist and can be accessed. It returns
|
||||
// ErrNoSourceData if none remain, or ErrInvalidSourceData if some were skipped.
|
||||
func filterExisting(items []string, warnf func(msg string, args ...interface{})) (result []string, err error) {
|
||||
for _, item := range items {
|
||||
_, err := fs.Lstat(item)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
warnf("%v does not exist, skipping\n", item)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
warnf("%v does not exist, skipping\n", item)
|
||||
} else {
|
||||
warnf("%v cannot be accessed, skipping\n", item)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
@@ -76,6 +77,28 @@ func TestCollectTargets(t *testing.T) {
|
||||
rtest.Assert(t, err == ErrInvalidSourceData, "expected error when not all targets exist")
|
||||
}
|
||||
|
||||
func TestFilterExistingUnreadable(t *testing.T) {
|
||||
dir := rtest.TempDir(t)
|
||||
|
||||
existing := filepath.Join(dir, "existing")
|
||||
rtest.OK(t, os.Mkdir(existing, 0755))
|
||||
|
||||
file := filepath.Join(dir, "file")
|
||||
rtest.OK(t, os.WriteFile(file, []byte("x"), 0600))
|
||||
|
||||
// Regression test for #5667. A target whose Lstat fails with an error other
|
||||
// than ErrNotExist must be skipped (ENOTDIR on unix, NUL byte everywhere).
|
||||
for _, unreadable := range []string{filepath.Join(file, "child"), "invalid\x00path"} {
|
||||
result, err := filterExisting([]string{unreadable}, t.Logf)
|
||||
rtest.Assert(t, errors.Is(err, ErrNoSourceData), "input %q: expected ErrNoSourceData; got %v", unreadable, err)
|
||||
rtest.Assert(t, len(result) == 0, "input %q: expected no targets; got %v", unreadable, result)
|
||||
|
||||
result, err = filterExisting([]string{existing, unreadable}, t.Logf)
|
||||
rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "input %q: expected ErrInvalidSourceData; got %v", unreadable, err)
|
||||
rtest.Equals(t, []string{existing}, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFilenamesRaw(t *testing.T) {
|
||||
// These should all be returned exactly as-is.
|
||||
expected := []string{
|
||||
|
||||
@@ -307,7 +307,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
|
||||
go chkr.Packs(ctx, errChan)
|
||||
|
||||
for err := range errChan {
|
||||
var packErr *repository.ErrPackMetadata
|
||||
var packErr *repository.PackError
|
||||
if errors.As(err, &packErr) {
|
||||
if packErr.Orphaned {
|
||||
orphanedPacks++
|
||||
|
||||
@@ -103,7 +103,7 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
|
||||
defer unlock()
|
||||
|
||||
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
|
||||
blobs, err := repo.ListPack(context.TODO(), id, size)
|
||||
blobs, _, err := repo.ListPack(context.TODO(), id, size)
|
||||
rtest.OK(t, err)
|
||||
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
|
||||
|
||||
|
||||
+365
-10
@@ -4,19 +4,31 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/data"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/global"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) {
|
||||
@@ -116,6 +128,61 @@ func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io
|
||||
})
|
||||
}
|
||||
|
||||
// Pack is the struct used in printPacks.
|
||||
type Pack struct {
|
||||
Name string `json:"name"`
|
||||
|
||||
Blobs []Blob `json:"blobs"`
|
||||
}
|
||||
|
||||
// Blob is the struct used in printPacks.
|
||||
type Blob struct {
|
||||
Type restic.BlobType `json:"type"`
|
||||
Length uint `json:"length"`
|
||||
ID restic.ID `json:"id"`
|
||||
Offset uint `json:"offset"`
|
||||
}
|
||||
|
||||
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error {
|
||||
|
||||
var m sync.Mutex
|
||||
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
|
||||
blobs, _, err := repo.ListPack(ctx, id, size)
|
||||
if err != nil {
|
||||
printer.E("error for pack %v: %v", id.Str(), err)
|
||||
return nil
|
||||
}
|
||||
|
||||
p := Pack{
|
||||
Name: id.String(),
|
||||
Blobs: make([]Blob, len(blobs)),
|
||||
}
|
||||
for i, blob := range blobs {
|
||||
p.Blobs[i] = Blob{
|
||||
Type: blob.Type,
|
||||
Length: blob.Length,
|
||||
ID: blob.ID,
|
||||
Offset: blob.Offset,
|
||||
}
|
||||
}
|
||||
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return prettyPrintJSON(wr, p)
|
||||
})
|
||||
}
|
||||
|
||||
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
|
||||
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
|
||||
printer.S("index_id: %v", id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return idx.Dump(wr)
|
||||
})
|
||||
}
|
||||
|
||||
func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
|
||||
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
|
||||
|
||||
@@ -133,11 +200,11 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
|
||||
|
||||
switch tpe {
|
||||
case "indexes":
|
||||
return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
|
||||
return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
|
||||
case "snapshots":
|
||||
return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
|
||||
case "packs":
|
||||
return repository.DumpPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
|
||||
return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
|
||||
case "all":
|
||||
printer.S("snapshots:")
|
||||
err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
|
||||
@@ -146,7 +213,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
|
||||
}
|
||||
|
||||
printer.S("indexes:")
|
||||
err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
|
||||
err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -157,6 +224,225 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
|
||||
}
|
||||
}
|
||||
|
||||
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
|
||||
if bytewise {
|
||||
printer.S(" trying to repair blob by finding a broken byte")
|
||||
} else {
|
||||
printer.S(" trying to repair blob with single bit flip")
|
||||
}
|
||||
|
||||
ch := make(chan int)
|
||||
var wg errgroup.Group
|
||||
done := make(chan struct{})
|
||||
var fixed []byte
|
||||
var found bool
|
||||
|
||||
workers := runtime.GOMAXPROCS(0)
|
||||
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Go(func() error {
|
||||
// make a local copy of the buffer
|
||||
buf := make([]byte, len(input))
|
||||
copy(buf, input)
|
||||
|
||||
testFlip := func(idx int, pattern byte) bool {
|
||||
// flip bits
|
||||
buf[idx] ^= pattern
|
||||
|
||||
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
||||
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
|
||||
if err == nil {
|
||||
printer.S("")
|
||||
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
|
||||
printer.S(" hash is %v", restic.Hash(plaintext))
|
||||
close(done)
|
||||
found = true
|
||||
fixed = plaintext
|
||||
return true
|
||||
}
|
||||
|
||||
// flip bits back
|
||||
buf[idx] ^= pattern
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range ch {
|
||||
if bytewise {
|
||||
for j := 0; j < 255; j++ {
|
||||
if testFlip(i, byte(j)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for j := 0; j < 7; j++ {
|
||||
// flip each bit once
|
||||
if testFlip(i, (1 << uint(j))) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
wg.Go(func() error {
|
||||
defer close(ch)
|
||||
|
||||
start := time.Now()
|
||||
info := time.Now()
|
||||
for i := range input {
|
||||
select {
|
||||
case ch <- i:
|
||||
case <-done:
|
||||
printer.S(" done after %v", time.Since(start))
|
||||
return nil
|
||||
}
|
||||
|
||||
if time.Since(info) > time.Second {
|
||||
secs := time.Since(start).Seconds()
|
||||
gps := float64(i) / secs
|
||||
remaining := len(input) - i
|
||||
eta := time.Duration(float64(remaining)/gps) * time.Second
|
||||
|
||||
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
|
||||
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
|
||||
info = time.Now()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
err := wg.Wait()
|
||||
if err != nil {
|
||||
panic("all go routines can only return nil")
|
||||
}
|
||||
|
||||
if !found {
|
||||
printer.S("\n blob could not be repaired")
|
||||
}
|
||||
return fixed
|
||||
}
|
||||
|
||||
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
|
||||
// strip signature at the end
|
||||
l := len(buf)
|
||||
nonce, ct := buf[:16], buf[16:l-16]
|
||||
out := make([]byte, len(ct))
|
||||
|
||||
c, err := aes.NewCipher(k.EncryptionKey[:])
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("unable to create cipher: %v", err))
|
||||
}
|
||||
e := cipher.NewCTR(c, nonce)
|
||||
e.XORKeyStream(out, ct)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
|
||||
dec, err := zstd.NewReader(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
|
||||
// allow processing broken pack files
|
||||
if pack == nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
for _, blob := range list {
|
||||
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
|
||||
if int(blob.Offset+blob.Length) > len(pack) {
|
||||
printer.E("skipping truncated blob")
|
||||
continue
|
||||
}
|
||||
buf := pack[blob.Offset : blob.Offset+blob.Length]
|
||||
key := repo.Key()
|
||||
|
||||
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
||||
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
|
||||
outputPrefix := ""
|
||||
filePrefix := ""
|
||||
if err != nil {
|
||||
printer.E("error decrypting blob: %v", err)
|
||||
if opts.TryRepair || opts.RepairByte {
|
||||
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
|
||||
}
|
||||
if plaintext != nil {
|
||||
outputPrefix = "repaired "
|
||||
filePrefix = "repaired-"
|
||||
} else {
|
||||
plaintext = decryptUnsigned(key, buf)
|
||||
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if blob.IsCompressed() {
|
||||
decompressed, err := dec.DecodeAll(plaintext, nil)
|
||||
if err != nil {
|
||||
printer.S(" failed to decompress blob %v", blob.ID)
|
||||
}
|
||||
if decompressed != nil {
|
||||
plaintext = decompressed
|
||||
}
|
||||
}
|
||||
|
||||
id := restic.Hash(plaintext)
|
||||
var prefix string
|
||||
if !id.Equal(blob.ID) {
|
||||
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
|
||||
prefix = "wrong-hash-"
|
||||
} else {
|
||||
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
|
||||
prefix = "correct-"
|
||||
}
|
||||
if opts.ExtractPack {
|
||||
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if opts.ReuploadBlobs {
|
||||
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printer.S(" uploaded %v %v", blob.Type, id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
|
||||
filename := fmt.Sprintf("%s%s.bin", prefix, id)
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = f.Write(plain)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printer.S("decrypt of blob %v stored at %v", id, filename)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
|
||||
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
|
||||
|
||||
@@ -192,14 +478,8 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
|
||||
return err
|
||||
}
|
||||
|
||||
examineOpts := repository.ExaminePackOptions{
|
||||
TryRepair: opts.TryRepair,
|
||||
RepairByte: opts.RepairByte,
|
||||
ExtractPack: opts.ExtractPack,
|
||||
ReuploadBlobs: opts.ReuploadBlobs,
|
||||
}
|
||||
for _, id := range ids {
|
||||
err := repository.ExaminePack(ctx, repo, id, examineOpts, printer)
|
||||
err := examinePack(ctx, opts, repo, id, printer)
|
||||
if err != nil {
|
||||
printer.E("error: %v", err)
|
||||
}
|
||||
@@ -209,3 +489,78 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
|
||||
printer.S("examine %v", id)
|
||||
|
||||
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
|
||||
// also process damaged pack files
|
||||
if buf == nil {
|
||||
return err
|
||||
}
|
||||
printer.S(" file size is %v", len(buf))
|
||||
gotID := restic.Hash(buf)
|
||||
if !id.Equal(gotID) {
|
||||
printer.S(" wanted hash %v, got %v", id, gotID)
|
||||
} else {
|
||||
printer.S(" hash for file content matches")
|
||||
}
|
||||
|
||||
printer.S(" ========================================")
|
||||
printer.S(" looking for info in the indexes")
|
||||
|
||||
blobsLoaded := false
|
||||
// examine all data the indexes have for the pack file
|
||||
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
|
||||
blobs := b.Blobs
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
checkPackSize(blobs, len(buf), printer)
|
||||
|
||||
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
|
||||
if err != nil {
|
||||
printer.E("error: %v", err)
|
||||
} else {
|
||||
blobsLoaded = true
|
||||
}
|
||||
}
|
||||
|
||||
printer.S(" ========================================")
|
||||
printer.S(" inspect the pack itself")
|
||||
|
||||
blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("pack %v: %v", id.Str(), err)
|
||||
}
|
||||
checkPackSize(blobs, len(buf), printer)
|
||||
|
||||
if !blobsLoaded {
|
||||
return loadBlobs(ctx, opts, repo, id, blobs, printer)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
|
||||
// track current size and offset
|
||||
var size, offset uint64
|
||||
|
||||
blobs.Sort()
|
||||
|
||||
for _, pb := range blobs {
|
||||
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
|
||||
if offset != uint64(pb.Offset) {
|
||||
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
|
||||
}
|
||||
offset = uint64(pb.Offset + pb.Length)
|
||||
size += uint64(pb.Length)
|
||||
}
|
||||
size += uint64(pack.CalculateHeaderSize(blobs))
|
||||
|
||||
if uint64(fileSize) != size {
|
||||
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
|
||||
} else {
|
||||
printer.S(" file sizes match")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -472,7 +472,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
||||
delete(packIDs, idStr)
|
||||
}
|
||||
debug.Log("Found pack %s", idStr)
|
||||
blobs, err := f.repo.ListPack(ctx, id, size)
|
||||
blobs, _, err := f.repo.ListPack(ctx, id, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -261,7 +261,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
|
||||
}
|
||||
|
||||
var key data.SnapshotGroupKey
|
||||
if json.Unmarshal([]byte(k), &key) != nil {
|
||||
if err := json.Unmarshal([]byte(k), &key); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+12
-7
@@ -6,7 +6,7 @@ import (
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/global"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
|
||||
@@ -69,13 +69,18 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T
|
||||
case "locks":
|
||||
t = restic.LockFile
|
||||
case "blobs":
|
||||
for entry := range repository.AllIndexBlobs(ctx, repo, repo) {
|
||||
if entry.Error != nil {
|
||||
return entry.Error
|
||||
return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printer.S("%v %v", entry.Handle.Type, entry.Handle.ID)
|
||||
}
|
||||
return nil
|
||||
for blobs := range idx.Values() {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
printer.S("%v %v", blobs.Type, blobs.ID)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
default:
|
||||
return errors.Fatal("invalid type")
|
||||
}
|
||||
|
||||
+93
-16
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +14,8 @@ import (
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/restic/restic/internal/backend/local"
|
||||
"github.com/restic/restic/internal/backend/location"
|
||||
"github.com/restic/restic/internal/data"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
@@ -131,22 +134,8 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
|
||||
}
|
||||
|
||||
mountpoint := args[0]
|
||||
|
||||
// Check the existence of the mount point at the earliest stage to
|
||||
// prevent unnecessary computations while opening the repository.
|
||||
stat, err := os.Stat(mountpoint)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
printer.P("Mountpoint %s doesn't exist", mountpoint)
|
||||
return errors.Fatal("invalid mountpoint")
|
||||
} else if !stat.IsDir() {
|
||||
printer.P("Mountpoint %s is not a directory", mountpoint)
|
||||
return errors.Fatal("invalid mountpoint")
|
||||
}
|
||||
|
||||
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
|
||||
if err != nil {
|
||||
printer.P("Mountpoint %s is not writeable or not executable", mountpoint)
|
||||
return errors.Fatal("inaccessible mountpoint")
|
||||
if err := validateMountpoint(mountpoint, gopts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
debug.Log("start mount")
|
||||
@@ -230,3 +219,91 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func validateMountpoint(mountpoint string, gopts global.Options) error {
|
||||
// Check the existence of the mount point at the earliest stage to
|
||||
// prevent unnecessary computations while opening the repository.
|
||||
stat, err := os.Stat(mountpoint)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return errors.Fatal(fmt.Sprintf("mountpoint %s does not exist", mountpoint))
|
||||
} else if err != nil {
|
||||
return errors.Fatal(fmt.Sprintf("mountpoint %s is inaccessible: %v", mountpoint, err))
|
||||
} else if !stat.IsDir() {
|
||||
return errors.Fatal(fmt.Sprintf("mountpoint %s is not a directory", mountpoint))
|
||||
}
|
||||
|
||||
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
|
||||
if err != nil {
|
||||
return errors.Fatal(fmt.Sprintf("mountpoint %s is not writeable or not executable", mountpoint))
|
||||
}
|
||||
|
||||
// Refuse to mount onto (or under, or over) the local repository directory.
|
||||
// Doing so makes the FUSE server read its own backend files through the
|
||||
// mount it just created, deadlocking the kernel (GH #5234).
|
||||
loc, err := location.Parse(gopts.Backends, gopts.Repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if loc.Scheme == "local" {
|
||||
if err := checkMountpointOverlap(loc.Config.(*local.Config).Path, mountpoint); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkMountpointOverlap returns an error.Fatal if the local repository at
|
||||
// repoPath and the mountpoint overlap: equal paths, mountpoint nested inside
|
||||
// the repo, or the repo nested inside the mountpoint. Any overlap deadlocks
|
||||
// the FUSE server (GH #5234).
|
||||
func checkMountpointOverlap(repoPath, mountpoint string) error {
|
||||
rp, err := resolvePath(repoPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mp, err := resolvePath(mountpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
const tail = "; refusing to mount to avoid deadlocking the FUSE server"
|
||||
switch {
|
||||
case rp == mp:
|
||||
return errors.Fatal(fmt.Sprintf("mountpoint %s is the local repository directory%s", mp, tail))
|
||||
case isInside(rp, mp):
|
||||
return errors.Fatal(fmt.Sprintf("mountpoint %s is inside the local repository directory %s%s", mp, rp, tail))
|
||||
case isInside(mp, rp):
|
||||
return errors.Fatal(fmt.Sprintf("local repository directory %s is inside the mountpoint %s%s", rp, mp, tail))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolvePath returns p as an absolute, symlink-resolved path. If EvalSymlinks
|
||||
// fails (e.g. the path does not fully exist), it falls back to the absolute
|
||||
// form: overlap detection is best-effort and we'd rather refuse a clear
|
||||
// overlap than abort on an unrelated stat error.
|
||||
func resolvePath(p string) (string, error) {
|
||||
abs, err := filepath.Abs(p)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(abs)
|
||||
if err != nil {
|
||||
return abs, nil
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
// isInside reports whether child is strictly nested inside parent. Both paths
|
||||
// must already be cleaned and absolute. Equal paths return false; the caller
|
||||
// handles equality separately so it can produce a distinct error message.
|
||||
func isInside(parent, child string) bool {
|
||||
rel, err := filepath.Rel(parent, child)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if rel == "." || rel == ".." {
|
||||
return false
|
||||
}
|
||||
return !strings.HasPrefix(rel, ".."+string(filepath.Separator))
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -212,6 +213,62 @@ func TestMount(t *testing.T) {
|
||||
checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 4)
|
||||
}
|
||||
|
||||
func TestCheckMountpointOverlap(t *testing.T) {
|
||||
tempdir := t.TempDir()
|
||||
repo := filepath.Join(tempdir, "repo")
|
||||
repoSub := filepath.Join(repo, "sub")
|
||||
sibling := filepath.Join(tempdir, "mnt")
|
||||
for _, d := range []string{repo, repoSub, sibling} {
|
||||
rtest.OK(t, os.MkdirAll(d, 0700))
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
repo string
|
||||
mount string
|
||||
wantSub string // substring of expected error; empty means expect nil
|
||||
}{
|
||||
{"equal", repo, repo, "is the local repository directory"},
|
||||
{"mount inside repo", repo, repoSub, "is inside the local repository directory"},
|
||||
{"repo inside mount", repoSub, repo, "is inside the mountpoint"},
|
||||
{"disjoint", repo, sibling, ""},
|
||||
{"prefix-not-subpath", repo, repo + "-other", ""},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rtest.OK(t, os.MkdirAll(tc.mount, 0700))
|
||||
err := checkMountpointOverlap(tc.repo, tc.mount)
|
||||
if tc.wantSub == "" {
|
||||
rtest.OK(t, err)
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q, got nil", tc.wantSub)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantSub) {
|
||||
t.Fatalf("error %q does not contain %q", err.Error(), tc.wantSub)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckMountpointOverlapSymlink(t *testing.T) {
|
||||
tempdir := t.TempDir()
|
||||
repo := filepath.Join(tempdir, "repo")
|
||||
rtest.OK(t, os.MkdirAll(repo, 0700))
|
||||
link := filepath.Join(tempdir, "link-to-repo")
|
||||
rtest.OK(t, os.Symlink(repo, link))
|
||||
|
||||
err := checkMountpointOverlap(repo, link)
|
||||
if err == nil {
|
||||
t.Fatal("expected overlap error when mountpoint is a symlink to repo, got nil")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "is the local repository directory") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMountSameTimestamps(t *testing.T) {
|
||||
if !rtest.RunFuseTest {
|
||||
t.Skip("Skipping fuse tests")
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/global"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
@@ -60,6 +61,15 @@ func TestRebuildIndex(t *testing.T) {
|
||||
testRebuildIndex(t, nil)
|
||||
}
|
||||
|
||||
func TestRebuildIndexAlwaysFull(t *testing.T) {
|
||||
indexFull := index.Full
|
||||
defer func() {
|
||||
index.Full = indexFull
|
||||
}()
|
||||
index.Full = func(*index.Index) bool { return true }
|
||||
testRebuildIndex(t, nil)
|
||||
}
|
||||
|
||||
// indexErrorBackend modifies the first index after reading.
|
||||
type indexErrorBackend struct {
|
||||
backend.Backend
|
||||
|
||||
+59
-12
@@ -38,6 +38,9 @@ Exit status is 12 if the password is incorrect.
|
||||
`,
|
||||
GroupID: cmdGroupDefault,
|
||||
DisableAutoGenTag: true,
|
||||
PreRunE: func(_ *cobra.Command, _ []string) error {
|
||||
return opts.Finalize()
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
finalizeSnapshotFilter(&opts.SnapshotFilter)
|
||||
return runSnapshots(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
|
||||
@@ -52,7 +55,7 @@ Exit status is 12 if the password is incorrect.
|
||||
type SnapshotOptions struct {
|
||||
data.SnapshotFilter
|
||||
Compact bool
|
||||
Last bool // This option should be removed in favour of Latest.
|
||||
last bool // Deprecated in favour of Latest.
|
||||
Latest int
|
||||
GroupBy data.SnapshotGroupByOptions
|
||||
}
|
||||
@@ -60,7 +63,7 @@ type SnapshotOptions struct {
|
||||
func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
|
||||
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
|
||||
f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format")
|
||||
f.BoolVar(&opts.Last, "last", false, "only show the last snapshot for each host and path")
|
||||
f.BoolVar(&opts.last, "last", false, "only show the last snapshot for each host and path")
|
||||
err := f.MarkDeprecated("last", "use --latest 1")
|
||||
if err != nil {
|
||||
// MarkDeprecated only returns an error when the flag is not found
|
||||
@@ -70,6 +73,13 @@ func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
|
||||
f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma")
|
||||
}
|
||||
|
||||
func (opts *SnapshotOptions) Finalize() error {
|
||||
if opts.last && opts.Latest == 0 {
|
||||
opts.Latest = 1
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Options, args []string, term ui.Terminal) error {
|
||||
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
|
||||
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
|
||||
@@ -95,12 +105,12 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if opts.Last {
|
||||
// This branch should be removed in the same time
|
||||
// that --last.
|
||||
list = filterLatestSnapshotsInGroup(list, 1)
|
||||
} else if opts.Latest > 0 {
|
||||
list = filterLatestSnapshotsInGroup(list, opts.Latest)
|
||||
if opts.Latest > 0 {
|
||||
if grouped {
|
||||
list = filterLatestSnapshotsInGroup(list, opts.Latest)
|
||||
} else {
|
||||
list = filterLatestSnapshots(list, opts.Latest)
|
||||
}
|
||||
}
|
||||
sort.Sort(sort.Reverse(list))
|
||||
snapshotGroups[k] = list
|
||||
@@ -134,6 +144,43 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
|
||||
return nil
|
||||
}
|
||||
|
||||
// filterLastSnapshotsKey is used by FilterLastSnapshots.
|
||||
type filterLastSnapshotsKey struct {
|
||||
Hostname string
|
||||
JoinedPaths string
|
||||
}
|
||||
|
||||
// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
|
||||
func newFilterLastSnapshotsKey(sn *data.Snapshot) filterLastSnapshotsKey {
|
||||
// Shallow slice copy
|
||||
var paths = make([]string, len(sn.Paths))
|
||||
copy(paths, sn.Paths)
|
||||
sort.Strings(paths)
|
||||
return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
|
||||
}
|
||||
|
||||
// filterLatestSnapshots filters a list of snapshots to only return
|
||||
// the limit last entries for each hostname and path. If the snapshot
|
||||
// contains multiple paths, they will be joined and treated as one
|
||||
// item.
|
||||
func filterLatestSnapshots(list data.Snapshots, limit int) data.Snapshots {
|
||||
// Sort the snapshots so that the newer ones are listed first
|
||||
sort.SliceStable(list, func(i, j int) bool {
|
||||
return list[i].Time.After(list[j].Time)
|
||||
})
|
||||
|
||||
var results data.Snapshots
|
||||
seen := make(map[filterLastSnapshotsKey]int)
|
||||
for _, sn := range list {
|
||||
key := newFilterLastSnapshotsKey(sn)
|
||||
if seen[key] < limit {
|
||||
seen[key]++
|
||||
results = append(results, sn)
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// filterLatestSnapshotsInGroup filters a list of snapshots to only return
|
||||
// the `limit` last entries. It is assumed that the snapshot list only contains
|
||||
// one group of snapshots.
|
||||
@@ -245,11 +292,11 @@ func PrintSnapshots(stdout io.Writer, list data.Snapshots, reasons []data.KeepRe
|
||||
// Each snapshot can be registered in different timezones,
|
||||
// but we display them all in local timezone on this output.
|
||||
footer := fmt.Sprintf("%d snapshots", len(list))
|
||||
zoneName, _ := time.Now().Local().Zone()
|
||||
if zoneName != "" {
|
||||
footer = fmt.Sprintf("Timestamps shown in %s timezone\n%s", zoneName, footer)
|
||||
zoneName := time.Now().Local().Location().String()
|
||||
if zoneName == "Local" {
|
||||
zoneName = "local time"
|
||||
}
|
||||
tab.AddFooter(footer)
|
||||
tab.AddFooter(fmt.Sprintf("Timestamps shown in %s\n%s", zoneName, footer))
|
||||
|
||||
if multiline {
|
||||
// print an additional blank line between snapshots
|
||||
|
||||
@@ -35,10 +35,7 @@ func testRunSnapshots(t testing.TB, gopts global.Options) (newest *Snapshot, sna
|
||||
return
|
||||
}
|
||||
|
||||
func TestSnapshotsGroupByAndLatest(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
func snapshotsGroupTestData(t *testing.T, env *testEnvironment, keepPath bool) string {
|
||||
testSetupBackupData(t, env)
|
||||
// two backups on the same host but with different paths
|
||||
opts := BackupOptions{Host: "testhost", TimeStamp: time.Now().Format(time.DateTime)}
|
||||
@@ -46,9 +43,22 @@ func TestSnapshotsGroupByAndLatest(t *testing.T) {
|
||||
// Use later timestamp for second backup
|
||||
opts.TimeStamp = time.Now().Add(time.Second).Format(time.DateTime)
|
||||
snapshotsIDs := loadSnapshotMap(t, env.gopts)
|
||||
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0"}, opts, env.gopts)
|
||||
|
||||
targets := []string{"testdata/0"}
|
||||
if keepPath {
|
||||
targets = []string{"testdata"}
|
||||
}
|
||||
testRunBackup(t, filepath.Dir(env.testdata), targets, opts, env.gopts)
|
||||
_, secondSnapshotID := lastSnapshot(snapshotsIDs, loadSnapshotMap(t, env.gopts))
|
||||
|
||||
return secondSnapshotID
|
||||
}
|
||||
|
||||
func TestSnapshotsGroupByAndLatest(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
secondSnapshotID := snapshotsGroupTestData(t, env, false)
|
||||
buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
|
||||
gopts.JSON = true
|
||||
// only group by host but not path
|
||||
@@ -65,3 +75,21 @@ func TestSnapshotsGroupByAndLatest(t *testing.T) {
|
||||
rtest.Assert(t, len(snapshots[0].Snapshots) == 1, "expected only one latest snapshot, got %d", len(snapshots[0].Snapshots))
|
||||
rtest.Equals(t, snapshots[0].Snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
|
||||
}
|
||||
|
||||
func TestSnapshotsLatest(t *testing.T) {
|
||||
env, cleanup := withTestEnvironment(t)
|
||||
defer cleanup()
|
||||
|
||||
secondSnapshotID := snapshotsGroupTestData(t, env, true)
|
||||
|
||||
buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
|
||||
gopts.JSON = true
|
||||
opts := SnapshotOptions{Latest: 1}
|
||||
return runSnapshots(ctx, opts, gopts, []string{}, gopts.Term)
|
||||
})
|
||||
rtest.OK(t, err)
|
||||
snapshots := []Snapshot{}
|
||||
rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
|
||||
rtest.Assert(t, len(snapshots) == 1, "expected only one snapshot, got %d", len(snapshots))
|
||||
rtest.Equals(t, snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
|
||||
}
|
||||
|
||||
@@ -141,7 +141,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
|
||||
snapshots = append(snapshots, sn)
|
||||
}
|
||||
|
||||
statsProgress := newStatsProgress(term, uint64(len(snapshots)))
|
||||
statsProgress := newStatsProgress(term, !gopts.JSON, uint64(len(snapshots)))
|
||||
|
||||
updater := progress.NewUpdater(ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()), func(runtime time.Duration, final bool) {
|
||||
statsProgress.printProgress(runtime, final)
|
||||
@@ -375,6 +375,7 @@ type statsProgress struct {
|
||||
term ui.Terminal
|
||||
m sync.Mutex
|
||||
snapshotCount uint64
|
||||
show bool
|
||||
|
||||
processedSnapshotCount uint64
|
||||
processedFileCount uint64
|
||||
@@ -382,14 +383,18 @@ type statsProgress struct {
|
||||
processedSize uint64
|
||||
}
|
||||
|
||||
func newStatsProgress(term ui.Terminal, snapshotCount uint64) *statsProgress {
|
||||
func newStatsProgress(term ui.Terminal, show bool, snapshotCount uint64) *statsProgress {
|
||||
return &statsProgress{
|
||||
term: term,
|
||||
show: show,
|
||||
snapshotCount: snapshotCount,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statsProgress) printProgress(runtime time.Duration, final bool) {
|
||||
if !s.show {
|
||||
return
|
||||
}
|
||||
s.m.Lock()
|
||||
|
||||
progressBase := s.processedSnapshotCount
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestSizeHistogramString(t *testing.T) {
|
||||
func TestStatsProgress(t *testing.T) {
|
||||
term := &ui.MockTerminal{}
|
||||
|
||||
progress := newStatsProgress(term, 2)
|
||||
progress := newStatsProgress(term, true, 2)
|
||||
progress.printProgress(0*time.Second, false)
|
||||
rtest.Equals(t, []string{"[0:00] 0.00% 0 / 2 snapshots, 0 B"}, term.Output)
|
||||
|
||||
@@ -88,3 +88,12 @@ func TestStatsProgress(t *testing.T) {
|
||||
progress.printProgress(20*time.Second, true)
|
||||
rtest.Equals(t, []string{"[0:20] 100.00% 2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output)
|
||||
}
|
||||
|
||||
func TestStatsProgressJSON(t *testing.T) {
|
||||
term := &ui.MockTerminal{}
|
||||
|
||||
progress := newStatsProgress(term, false, 2)
|
||||
progress.printProgress(0*time.Second, false)
|
||||
// JSON output is not available yet, so just make sure to not break normal json output
|
||||
rtest.Equals(t, nil, term.Output)
|
||||
}
|
||||
|
||||
@@ -208,6 +208,12 @@ command needs to be in the ``PATH``. On macOS, you need `FUSE-T
|
||||
<https://www.fuse-t.org/>`__ or `FUSE for macOS <https://osxfuse.github.io/>`__.
|
||||
On FreeBSD, you may need to install FUSE and load the kernel module (``kldload fuse``).
|
||||
|
||||
.. note:: The mountpoint must not overlap the local repository directory.
|
||||
Using the repository directory itself, a subdirectory of it, or a parent
|
||||
of it as the mountpoint causes the FUSE server to read its own backend
|
||||
files through the new mount and deadlock the kernel. ``restic mount``
|
||||
detects this and refuses such mountpoints.
|
||||
|
||||
Restic supports storage and preservation of hard links. However, since
|
||||
hard links exist in the scope of a filesystem by definition, restoring
|
||||
hard links from a FUSE mount should be done by a program that preserves
|
||||
|
||||
@@ -320,6 +320,8 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
|
||||
finder := data.NewTreeFinder(previous)
|
||||
defer finder.Close()
|
||||
|
||||
var lastExcluded string
|
||||
|
||||
for _, name := range names {
|
||||
// test if context has been cancelled
|
||||
if ctx.Err() != nil {
|
||||
@@ -327,6 +329,12 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
|
||||
return futureNode{}, ctx.Err()
|
||||
}
|
||||
|
||||
if name == lastExcluded {
|
||||
// Skip duplicate directory entry if it was already excluded.
|
||||
// This avoids printing errors about duplicate directory entries even though the entry in question is ignored.
|
||||
continue
|
||||
}
|
||||
|
||||
pathname := arch.FS.Join(dir, name)
|
||||
oldNode, err := finder.Find(name)
|
||||
err = arch.error(pathname, err)
|
||||
@@ -348,6 +356,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
|
||||
}
|
||||
|
||||
if excluded {
|
||||
lastExcluded = name
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -894,6 +894,90 @@ func TestArchiverSaveDir(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type duplicateReaddirFS struct {
|
||||
fs.FS
|
||||
dir string
|
||||
names []string
|
||||
}
|
||||
|
||||
func (d *duplicateReaddirFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
|
||||
f, err := d.FS.OpenFile(name, flag, metadataOnly)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if name == d.dir {
|
||||
return &duplicateReaddirFile{File: f, names: d.names}, nil
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
type duplicateReaddirFile struct {
|
||||
fs.File
|
||||
names []string
|
||||
}
|
||||
|
||||
func (f *duplicateReaddirFile) Readdirnames(int) ([]string, error) {
|
||||
return append([]string(nil), f.names...), nil
|
||||
}
|
||||
|
||||
func TestArchiverSaveDirDuplicateExcludedEntry(t *testing.T) {
|
||||
const targetNodeName = "targetdir"
|
||||
|
||||
src := TestDir{
|
||||
"excluded": TestFile{Content: "skip me"},
|
||||
"keep": TestFile{Content: "keep me"},
|
||||
}
|
||||
tempdir, repo := prepareTempdirRepoSrc(t, src)
|
||||
|
||||
testFS := fs.Track{FS: &duplicateReaddirFS{
|
||||
FS: &fs.Local{},
|
||||
dir: ".",
|
||||
names: []string{"excluded", "excluded", "keep"},
|
||||
}}
|
||||
arch := New(repo, testFS, Options{})
|
||||
arch.summary = &Summary{}
|
||||
arch.Select = func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
|
||||
return filepath.Base(item) != "excluded"
|
||||
}
|
||||
arch.Error = func(item string, err error) error {
|
||||
t.Errorf("unexpected archiver error for %v: %v", item, err)
|
||||
return err
|
||||
}
|
||||
|
||||
back := rtest.Chdir(t, tempdir)
|
||||
defer back()
|
||||
|
||||
// duplicate node check in tree finder is only done if the previous tree is not nil
|
||||
previousTree, err := data.NewTreeNodeIterator(strings.NewReader(`{"nodes":[]}`))
|
||||
rtest.OK(t, err)
|
||||
|
||||
var treeID restic.ID
|
||||
err = repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
wg, ctx := errgroup.WithContext(ctx)
|
||||
arch.runWorkers(ctx, wg, uploader)
|
||||
meta, err := testFS.OpenFile(".", fs.O_NOFOLLOW, true)
|
||||
rtest.OK(t, err)
|
||||
ft, err := arch.saveDir(ctx, "/", ".", meta, previousTree, nil)
|
||||
rtest.OK(t, err)
|
||||
rtest.OK(t, meta.Close())
|
||||
|
||||
fnr := ft.take(ctx)
|
||||
node := fnr.node
|
||||
node.Name = targetNodeName
|
||||
treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{node})
|
||||
arch.stopWorkers()
|
||||
return wg.Wait()
|
||||
})
|
||||
rtest.OK(t, err)
|
||||
|
||||
TestEnsureTree(context.TODO(), t, "/", repo, treeID, TestDir{
|
||||
"targetdir": TestDir{
|
||||
"keep": TestFile{Content: "keep me"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func TestArchiverSaveDirIncremental(t *testing.T) {
|
||||
tempdir := rtest.TempDir(t)
|
||||
|
||||
|
||||
@@ -31,9 +31,11 @@ import (
|
||||
|
||||
// Backend stores data on an azure endpoint.
|
||||
type Backend struct {
|
||||
cfg Config
|
||||
container *azContainer.Client
|
||||
connections uint
|
||||
cfg Config
|
||||
container *azContainer.Client
|
||||
connections uint
|
||||
prefix string
|
||||
listMaxItems int
|
||||
layout.Layout
|
||||
|
||||
accessTier blob.AccessTier
|
||||
@@ -143,11 +145,12 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
}
|
||||
|
||||
be := &Backend{
|
||||
container: client,
|
||||
cfg: cfg,
|
||||
connections: cfg.Connections,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
accessTier: accessTier,
|
||||
container: client,
|
||||
cfg: cfg,
|
||||
connections: cfg.Connections,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
listMaxItems: defaultListMaxItems,
|
||||
accessTier: accessTier,
|
||||
}
|
||||
|
||||
return be, nil
|
||||
@@ -192,6 +195,11 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// SetListMaxItems sets the number of list items to load per request.
|
||||
func (be *Backend) SetListMaxItems(i int) {
|
||||
be.listMaxItems = i
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a not existing file.
|
||||
func (be *Backend) IsNotExist(err error) bool {
|
||||
return bloberror.HasCode(err, bloberror.BlobNotFound)
|
||||
@@ -223,6 +231,11 @@ func (be *Backend) Hasher() hash.Hash {
|
||||
return md5.New()
|
||||
}
|
||||
|
||||
// Path returns the path in the bucket that is used for this backend.
|
||||
func (be *Backend) Path() string {
|
||||
return be.prefix
|
||||
}
|
||||
|
||||
// useAccessTier determines whether to apply the configured access tier to a given file.
|
||||
// For archive access tier, only data files are stored using that class; metadata
|
||||
// must remain instantly accessible.
|
||||
@@ -406,7 +419,7 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
|
||||
prefix += "/"
|
||||
}
|
||||
|
||||
maxI := int32(defaultListMaxItems)
|
||||
maxI := int32(be.listMaxItems)
|
||||
|
||||
opts := &azContainer.ListBlobsFlatOptions{
|
||||
MaxResults: &maxI,
|
||||
|
||||
+22
-13
@@ -23,9 +23,10 @@ import (
|
||||
|
||||
// b2Backend is a backend which stores its data on Backblaze B2.
|
||||
type b2Backend struct {
|
||||
client *b2.Client
|
||||
bucket *b2.Bucket
|
||||
cfg Config
|
||||
client *b2.Client
|
||||
bucket *b2.Bucket
|
||||
cfg Config
|
||||
listMaxItems int
|
||||
layout.Layout
|
||||
|
||||
canDelete bool
|
||||
@@ -106,11 +107,12 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string,
|
||||
}
|
||||
|
||||
be := &b2Backend{
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
cfg: cfg,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
canDelete: true,
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
cfg: cfg,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
listMaxItems: defaultListMaxItems,
|
||||
canDelete: true,
|
||||
}
|
||||
|
||||
return be, nil
|
||||
@@ -138,14 +140,20 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
|
||||
}
|
||||
|
||||
be := &b2Backend{
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
cfg: cfg,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
cfg: cfg,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
listMaxItems: defaultListMaxItems,
|
||||
}
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// SetListMaxItems sets the number of list items to load per request.
|
||||
func (be *b2Backend) SetListMaxItems(i int) {
|
||||
be.listMaxItems = i
|
||||
}
|
||||
|
||||
func (be *b2Backend) Properties() backend.Properties {
|
||||
return backend.Properties{
|
||||
Connections: be.cfg.Connections,
|
||||
@@ -238,6 +246,7 @@ func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewi
|
||||
|
||||
// sanity check
|
||||
if n != rd.Length() {
|
||||
_ = w.Close()
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
|
||||
}
|
||||
return errors.Wrap(w.Close(), "Close")
|
||||
@@ -296,7 +305,7 @@ func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backe
|
||||
defer cancel()
|
||||
|
||||
prefix, _ := be.Basedir(t)
|
||||
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(defaultListMaxItems))
|
||||
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems))
|
||||
|
||||
for iter.Next() {
|
||||
obj := iter.Object()
|
||||
|
||||
Vendored
+26
-18
@@ -7,10 +7,11 @@ import (
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// cacheBackend wraps a restic.cacheBackend and adds a cache.
|
||||
type cacheBackend struct {
|
||||
// Backend wraps a restic.Backend and adds a cache.
|
||||
type Backend struct {
|
||||
backend.Backend
|
||||
*Cache
|
||||
|
||||
@@ -23,10 +24,10 @@ type cacheBackend struct {
|
||||
}
|
||||
|
||||
// ensure Backend implements backend.Backend
|
||||
var _ backend.Backend = &cacheBackend{}
|
||||
var _ backend.Backend = &Backend{}
|
||||
|
||||
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *cacheBackend {
|
||||
return &cacheBackend{
|
||||
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *Backend {
|
||||
return &Backend{
|
||||
Backend: be,
|
||||
Cache: c,
|
||||
inProgress: make(map[backend.Handle]chan struct{}),
|
||||
@@ -35,7 +36,7 @@ func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface
|
||||
}
|
||||
|
||||
// Remove deletes a file from the backend and the cache if it has been cached.
|
||||
func (b *cacheBackend) Remove(ctx context.Context, h backend.Handle) error {
|
||||
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
|
||||
debug.Log("cache Remove(%v)", h)
|
||||
err := b.Backend.Remove(ctx, h)
|
||||
if err != nil {
|
||||
@@ -57,7 +58,7 @@ func autoCacheTypes(h backend.Handle) bool {
|
||||
}
|
||||
|
||||
// Save stores a new file in the backend and the cache.
|
||||
func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
|
||||
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
|
||||
if !autoCacheTypes(h) {
|
||||
return b.Backend.Save(ctx, h, rd)
|
||||
}
|
||||
@@ -91,7 +92,7 @@ func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.Re
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
|
||||
func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
|
||||
finish := make(chan struct{})
|
||||
|
||||
b.inProgressMutex.Lock()
|
||||
@@ -135,7 +136,7 @@ func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
|
||||
}
|
||||
|
||||
// loadFromCache will try to load the file from the cache.
|
||||
func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
|
||||
func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
|
||||
rd, inCache, err := b.Cache.load(h, length, offset)
|
||||
if err != nil {
|
||||
return inCache, err
|
||||
@@ -150,7 +151,7 @@ func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64,
|
||||
}
|
||||
|
||||
// Load loads a file from the cache or the backend.
|
||||
func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
|
||||
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
|
||||
b.inProgressMutex.Lock()
|
||||
waitForFinish, inProgress := b.inProgress[h]
|
||||
b.inProgressMutex.Unlock()
|
||||
@@ -197,7 +198,7 @@ func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, o
|
||||
|
||||
// Stat tests whether the backend has a file. If it does not exist but still
|
||||
// exists in the cache, it is removed from the cache.
|
||||
func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
|
||||
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
|
||||
debug.Log("cache Stat(%v)", h)
|
||||
|
||||
fi, err := b.Backend.Stat(ctx, h)
|
||||
@@ -210,24 +211,31 @@ func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.File
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a non-existing file.
|
||||
func (b *cacheBackend) IsNotExist(err error) bool {
|
||||
func (b *Backend) IsNotExist(err error) bool {
|
||||
return b.Backend.IsNotExist(err)
|
||||
}
|
||||
|
||||
func (b *cacheBackend) Unwrap() backend.Backend {
|
||||
func (b *Backend) Unwrap() backend.Backend {
|
||||
return b.Backend
|
||||
}
|
||||
|
||||
func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
|
||||
func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
|
||||
if !b.Cache.canBeCached(t) {
|
||||
return b.Backend.List(ctx, t, fn)
|
||||
}
|
||||
|
||||
ids := make(map[string]struct{})
|
||||
// will contain the IDs of the files that are in the repository
|
||||
ids := restic.NewIDSet()
|
||||
|
||||
// wrap the original function to also add the file to the ids set
|
||||
wrapFn := func(f backend.FileInfo) error {
|
||||
ids[f.Name] = struct{}{}
|
||||
id, err := restic.ParseID(f.Name)
|
||||
if err != nil {
|
||||
// ignore files with invalid name
|
||||
return nil
|
||||
}
|
||||
|
||||
ids.Insert(id)
|
||||
|
||||
// execute the original function
|
||||
return fn(f)
|
||||
@@ -252,11 +260,11 @@ func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f b
|
||||
}
|
||||
|
||||
// Warmup delegates to wrapped backend.
|
||||
func (b *cacheBackend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
|
||||
func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
|
||||
return b.Backend.Warmup(ctx, h)
|
||||
}
|
||||
|
||||
// WarmupWait delegates to wrapped backend.
|
||||
func (b *cacheBackend) WarmupWait(ctx context.Context, h []backend.Handle) error {
|
||||
func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error {
|
||||
return b.Backend.WarmupWait(ctx, h)
|
||||
}
|
||||
|
||||
Vendored
+5
-4
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Cache manages a local cache.
|
||||
@@ -42,10 +43,10 @@ func readVersion(dir string) (v uint, err error) {
|
||||
|
||||
const cacheVersion = 1
|
||||
|
||||
var cacheLayoutPaths = map[backend.FileType]string{
|
||||
backend.PackFile: "data",
|
||||
backend.SnapshotFile: "snapshots",
|
||||
backend.IndexFile: "index",
|
||||
var cacheLayoutPaths = map[restic.FileType]string{
|
||||
restic.PackFile: "data",
|
||||
restic.SnapshotFile: "snapshots",
|
||||
restic.IndexFile: "index",
|
||||
}
|
||||
|
||||
const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"
|
||||
|
||||
Vendored
+12
-7
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/restic/restic/internal/backend/util"
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
func (c *Cache) filename(h backend.Handle) string {
|
||||
@@ -170,7 +171,7 @@ func (c *Cache) remove(h backend.Handle) (bool, error) {
|
||||
|
||||
// Clear removes all files of type t from the cache that are not contained in
|
||||
// the set valid.
|
||||
func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
|
||||
func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
|
||||
debug.Log("Clearing cache for %v: %v valid files", t, len(valid))
|
||||
if !c.canBeCached(t) {
|
||||
return nil
|
||||
@@ -182,12 +183,12 @@ func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
|
||||
}
|
||||
|
||||
for id := range list {
|
||||
if _, ok := valid[id]; ok {
|
||||
if valid.Has(id) {
|
||||
continue
|
||||
}
|
||||
|
||||
// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
|
||||
if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id})); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -200,12 +201,12 @@ func isFile(fi os.FileInfo) bool {
|
||||
}
|
||||
|
||||
// list returns a list of all files of type T in the cache.
|
||||
func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
|
||||
func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
|
||||
if !c.canBeCached(t) {
|
||||
return nil, errors.New("cannot be cached")
|
||||
}
|
||||
|
||||
list := make(map[string]struct{})
|
||||
list := restic.NewIDSet()
|
||||
dir := filepath.Join(c.path, cacheLayoutPaths[t])
|
||||
err := filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
@@ -220,8 +221,12 @@ func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
id := filepath.Base(name)
|
||||
list[id] = struct{}{}
|
||||
id, err := restic.ParseID(filepath.Base(name))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
list.Insert(id)
|
||||
return nil
|
||||
})
|
||||
|
||||
|
||||
Vendored
+18
-19
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"math/rand"
|
||||
"os"
|
||||
"runtime"
|
||||
@@ -19,8 +18,8 @@ import (
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} {
|
||||
ids := make(map[string]struct{})
|
||||
func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet {
|
||||
ids := restic.NewIDSet()
|
||||
for i := 0; i < random.Intn(15)+10; i++ {
|
||||
buf := rtest.Random(random.Int(), 1<<19)
|
||||
id := restic.Hash(buf)
|
||||
@@ -34,13 +33,13 @@ func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType,
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids[id.String()] = struct{}{}
|
||||
ids.Insert(id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// randomID returns a random ID from s.
|
||||
func randomID(s map[string]struct{}) string {
|
||||
func randomID(s restic.IDSet) restic.ID {
|
||||
for id := range s {
|
||||
return id
|
||||
}
|
||||
@@ -70,7 +69,7 @@ func load(t testing.TB, c *Cache, h backend.Handle) []byte {
|
||||
return buf
|
||||
}
|
||||
|
||||
func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{} {
|
||||
func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
|
||||
list, err := c.list(tpe)
|
||||
if err != nil {
|
||||
t.Errorf("listing failed: %v", err)
|
||||
@@ -79,7 +78,7 @@ func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{}
|
||||
return list
|
||||
}
|
||||
|
||||
func clearFiles(t testing.TB, c *Cache, tpe backend.FileType, valid map[string]struct{}) {
|
||||
func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
|
||||
if err := c.Clear(tpe, valid); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -103,34 +102,34 @@ func TestFiles(t *testing.T) {
|
||||
ids := generateRandomFiles(t, random, tpe, c)
|
||||
id := randomID(ids)
|
||||
|
||||
h := backend.Handle{Type: tpe, Name: id}
|
||||
h := backend.Handle{Type: tpe, Name: id.String()}
|
||||
id2 := restic.Hash(load(t, c, h))
|
||||
|
||||
if id != id2.String() {
|
||||
t.Errorf("wrong data returned, want %v, got %v", id, id2.String())
|
||||
if !id.Equal(id2) {
|
||||
t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
|
||||
}
|
||||
|
||||
if !c.Has(h) {
|
||||
t.Errorf("cache thinks index %v isn't present", id)
|
||||
t.Errorf("cache thinks index %v isn't present", id.Str())
|
||||
}
|
||||
|
||||
list := listFiles(t, c, tpe)
|
||||
if !maps.Equal(ids, list) {
|
||||
if !ids.Equals(list) {
|
||||
t.Errorf("wrong list of index IDs returned, want:\n %v\ngot:\n %v", ids, list)
|
||||
}
|
||||
|
||||
clearFiles(t, c, tpe, map[string]struct{}{id: {}})
|
||||
clearFiles(t, c, tpe, restic.NewIDSet(id))
|
||||
list2 := listFiles(t, c, tpe)
|
||||
delete(ids, id)
|
||||
want := map[string]struct{}{id: {}}
|
||||
if !maps.Equal(list2, want) {
|
||||
ids.Delete(id)
|
||||
want := restic.NewIDSet(id)
|
||||
if !list2.Equals(want) {
|
||||
t.Errorf("ClearIndexes removed indexes, want:\n %v\ngot:\n %v", list2, want)
|
||||
}
|
||||
|
||||
clearFiles(t, c, tpe, map[string]struct{}{})
|
||||
want = map[string]struct{}{}
|
||||
clearFiles(t, c, tpe, restic.NewIDSet())
|
||||
want = restic.NewIDSet()
|
||||
list3 := listFiles(t, c, tpe)
|
||||
if !maps.Equal(list3, want) {
|
||||
if !list3.Equals(want) {
|
||||
t.Errorf("ClearIndexes returned a wrong list, want:\n %v\ngot:\n %v", want, list3)
|
||||
}
|
||||
})
|
||||
|
||||
Vendored
+2
-3
@@ -3,17 +3,16 @@ package cache
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
const testCacheID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
// TestNewCache returns a cache in a temporary directory which is removed when
|
||||
// cleanup is called.
|
||||
func TestNewCache(t testing.TB) *Cache {
|
||||
dir := test.TempDir(t)
|
||||
t.Logf("created new cache at %v", dir)
|
||||
cache, err := New(testCacheID, dir)
|
||||
cache, err := New(restic.NewRandomID().String(), dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+49
-33
@@ -27,25 +27,27 @@ import (
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
// gs stores data in a GCS bucket.
|
||||
// Backend stores data in a GCS bucket.
|
||||
//
|
||||
// The service account used to access the bucket must have these permissions:
|
||||
// - storage.objects.create
|
||||
// - storage.objects.delete
|
||||
// - storage.objects.get
|
||||
// - storage.objects.list
|
||||
type gs struct {
|
||||
gcsClient *storage.Client
|
||||
projectID string
|
||||
connections uint
|
||||
bucketName string
|
||||
region string
|
||||
bucket *storage.BucketHandle
|
||||
type Backend struct {
|
||||
gcsClient *storage.Client
|
||||
projectID string
|
||||
connections uint
|
||||
bucketName string
|
||||
region string
|
||||
bucket *storage.BucketHandle
|
||||
prefix string
|
||||
listMaxItems int
|
||||
layout.Layout
|
||||
}
|
||||
|
||||
// Ensure that *Backend implements backend.Backend.
|
||||
var _ backend.Backend = &gs{}
|
||||
var _ backend.Backend = &Backend{}
|
||||
|
||||
func NewFactory() location.Factory {
|
||||
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
|
||||
@@ -84,7 +86,7 @@ func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
|
||||
return gcsClient, nil
|
||||
}
|
||||
|
||||
func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
|
||||
func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
|
||||
_, err := bucket.Attrs(ctx)
|
||||
if err == storage.ErrBucketNotExist {
|
||||
return false, nil
|
||||
@@ -92,7 +94,9 @@ func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (b
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
func open(cfg Config, rt http.RoundTripper) (*gs, error) {
|
||||
const defaultListMaxItems = 1000
|
||||
|
||||
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
debug.Log("open, config %#v", cfg)
|
||||
|
||||
gcsClient, err := getStorageClient(rt)
|
||||
@@ -100,14 +104,16 @@ func open(cfg Config, rt http.RoundTripper) (*gs, error) {
|
||||
return nil, errors.Wrap(err, "getStorageClient")
|
||||
}
|
||||
|
||||
be := &gs{
|
||||
gcsClient: gcsClient,
|
||||
projectID: cfg.ProjectID,
|
||||
connections: cfg.Connections,
|
||||
bucketName: cfg.Bucket,
|
||||
region: cfg.Region,
|
||||
bucket: gcsClient.Bucket(cfg.Bucket),
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
be := &Backend{
|
||||
gcsClient: gcsClient,
|
||||
projectID: cfg.ProjectID,
|
||||
connections: cfg.Connections,
|
||||
bucketName: cfg.Bucket,
|
||||
region: cfg.Region,
|
||||
bucket: gcsClient.Bucket(cfg.Bucket),
|
||||
prefix: cfg.Prefix,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
listMaxItems: defaultListMaxItems,
|
||||
}
|
||||
|
||||
return be, nil
|
||||
@@ -155,12 +161,17 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// SetListMaxItems sets the number of list items to load per request.
|
||||
func (be *Backend) SetListMaxItems(i int) {
|
||||
be.listMaxItems = i
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a not existing file.
|
||||
func (be *gs) IsNotExist(err error) bool {
|
||||
func (be *Backend) IsNotExist(err error) bool {
|
||||
return errors.Is(err, storage.ErrObjectNotExist)
|
||||
}
|
||||
|
||||
func (be *gs) IsPermanentError(err error) bool {
|
||||
func (be *Backend) IsPermanentError(err error) bool {
|
||||
if be.IsNotExist(err) {
|
||||
return true
|
||||
}
|
||||
@@ -175,7 +186,7 @@ func (be *gs) IsPermanentError(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (be *gs) Properties() backend.Properties {
|
||||
func (be *Backend) Properties() backend.Properties {
|
||||
return backend.Properties{
|
||||
Connections: be.connections,
|
||||
HasAtomicReplace: true,
|
||||
@@ -183,12 +194,17 @@ func (be *gs) Properties() backend.Properties {
|
||||
}
|
||||
|
||||
// Hasher may return a hash function for calculating a content hash for the backend
|
||||
func (be *gs) Hasher() hash.Hash {
|
||||
func (be *Backend) Hasher() hash.Hash {
|
||||
return md5.New()
|
||||
}
|
||||
|
||||
// Path returns the path in the bucket that is used for this backend.
|
||||
func (be *Backend) Path() string {
|
||||
return be.prefix
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
|
||||
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
|
||||
objName := be.Filename(h)
|
||||
|
||||
// Set chunk size to zero to disable resumable uploads.
|
||||
@@ -238,14 +254,14 @@ func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReade
|
||||
|
||||
// Load runs fn with a reader that yields the contents of the file at h at the
|
||||
// given offset.
|
||||
func (be *gs) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
|
||||
}
|
||||
|
||||
func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
if length == 0 {
|
||||
// negative length indicates read till end to GCS lib
|
||||
length = -1
|
||||
@@ -267,7 +283,7 @@ func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offs
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
|
||||
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
|
||||
objName := be.Filename(h)
|
||||
|
||||
attr, err := be.bucket.Object(objName).Attrs(ctx)
|
||||
@@ -280,7 +296,7 @@ func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo,
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
|
||||
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
|
||||
objName := be.Filename(h)
|
||||
|
||||
err := be.bucket.Object(objName).Delete(ctx)
|
||||
@@ -294,7 +310,7 @@ func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
|
||||
|
||||
// List runs fn for each file in the backend which has the type t. When an
|
||||
// error occurs (or fn returns an error), List stops and returns it.
|
||||
func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
|
||||
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
|
||||
prefix, _ := be.Basedir(t)
|
||||
|
||||
// make sure prefix ends with a slash
|
||||
@@ -339,15 +355,15 @@ func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.File
|
||||
}
|
||||
|
||||
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
|
||||
func (be *gs) Delete(ctx context.Context) error {
|
||||
func (be *Backend) Delete(ctx context.Context) error {
|
||||
return util.DefaultDelete(ctx, be)
|
||||
}
|
||||
|
||||
// Close does nothing.
|
||||
func (be *gs) Close() error { return nil }
|
||||
func (be *Backend) Close() error { return nil }
|
||||
|
||||
// Warmup not implemented
|
||||
func (be *gs) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
|
||||
func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
|
||||
return []backend.Handle{}, nil
|
||||
}
|
||||
func (be *gs) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
|
||||
func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
|
||||
|
||||
@@ -28,8 +28,8 @@ import (
|
||||
"golang.org/x/net/http2"
|
||||
)
|
||||
|
||||
// rclone is used to access data stored somewhere via rclone.
|
||||
type rclone struct {
|
||||
// Backend is used to access data stored somewhere via rclone.
|
||||
type Backend struct {
|
||||
*rest.Backend
|
||||
tr *http2.Transport
|
||||
cmd *exec.Cmd
|
||||
@@ -141,7 +141,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn {
|
||||
}
|
||||
|
||||
// New initializes a Backend and starts the process.
|
||||
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*rclone, error) {
|
||||
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
|
||||
var (
|
||||
args []string
|
||||
err error
|
||||
@@ -196,7 +196,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
|
||||
}
|
||||
|
||||
cmd := stdioConn.cmd
|
||||
be := &rclone{
|
||||
be := &Backend{
|
||||
tr: tr,
|
||||
cmd: cmd,
|
||||
waitCh: waitCh,
|
||||
@@ -270,7 +270,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
|
||||
}
|
||||
|
||||
// Open starts an rclone process with the given config.
|
||||
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
|
||||
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
|
||||
be, err := newBackend(ctx, cfg, lim, errorLog)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -297,7 +297,7 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(st
|
||||
}
|
||||
|
||||
// Create initializes a new restic repo with rclone.
|
||||
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
|
||||
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
|
||||
be, err := newBackend(ctx, cfg, lim, errorLog)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -328,7 +328,7 @@ func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(
|
||||
const waitForExit = 5 * time.Second
|
||||
|
||||
// Close terminates the backend.
|
||||
func (be *rclone) Close() error {
|
||||
func (be *Backend) Close() error {
|
||||
debug.Log("exiting rclone")
|
||||
be.tr.CloseIdleConnections()
|
||||
|
||||
@@ -348,7 +348,7 @@ func (be *rclone) Close() error {
|
||||
return be.waitResult
|
||||
}
|
||||
|
||||
func (be *rclone) Properties() backend.Properties {
|
||||
func (be *Backend) Properties() backend.Properties {
|
||||
properties := be.Backend.Properties()
|
||||
properties.HasFlakyErrors = true
|
||||
return properties
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestRcloneExit(t *testing.T) {
|
||||
_ = be.Close()
|
||||
}()
|
||||
|
||||
err = be.(*rclone).cmd.Process.Kill()
|
||||
err = be.cmd.Process.Kill()
|
||||
rtest.OK(t, err)
|
||||
t.Log("killed rclone")
|
||||
|
||||
|
||||
@@ -245,6 +245,7 @@ func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int,
|
||||
}
|
||||
|
||||
if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 && resp.ContentLength != int64(length) {
|
||||
_ = drainAndClose(resp)
|
||||
return nil, &restError{h, http.StatusRequestedRangeNotSatisfiable, "partial out of bounds read"}
|
||||
}
|
||||
|
||||
|
||||
+28
-23
@@ -25,15 +25,15 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// s3 stores data on an S3 endpoint.
|
||||
type s3 struct {
|
||||
// Backend stores data on an S3 endpoint.
|
||||
type Backend struct {
|
||||
client *minio.Client
|
||||
cfg Config
|
||||
layout.Layout
|
||||
}
|
||||
|
||||
// make sure that *Backend implements backend.Backend
|
||||
var _ backend.Backend = &s3{}
|
||||
var _ backend.Backend = &Backend{}
|
||||
|
||||
var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"}
|
||||
|
||||
@@ -50,7 +50,7 @@ func NewFactory() location.Factory {
|
||||
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
|
||||
}
|
||||
|
||||
func open(cfg Config, rt http.RoundTripper) (*s3, error) {
|
||||
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
|
||||
debug.Log("open, config %#v", cfg)
|
||||
|
||||
if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) {
|
||||
@@ -89,7 +89,7 @@ func open(cfg Config, rt http.RoundTripper) (*s3, error) {
|
||||
return nil, errors.Wrap(err, "minio.New")
|
||||
}
|
||||
|
||||
be := &s3{
|
||||
be := &Backend{
|
||||
client: client,
|
||||
cfg: cfg,
|
||||
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
|
||||
@@ -240,12 +240,12 @@ func isAccessDenied(err error) bool {
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a not existing file.
|
||||
func (be *s3) IsNotExist(err error) bool {
|
||||
func (be *Backend) IsNotExist(err error) bool {
|
||||
var e minio.ErrorResponse
|
||||
return errors.As(err, &e) && e.Code == "NoSuchKey"
|
||||
}
|
||||
|
||||
func (be *s3) IsPermanentError(err error) bool {
|
||||
func (be *Backend) IsPermanentError(err error) bool {
|
||||
if be.IsNotExist(err) {
|
||||
return true
|
||||
}
|
||||
@@ -260,7 +260,7 @@ func (be *s3) IsPermanentError(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (be *s3) Properties() backend.Properties {
|
||||
func (be *Backend) Properties() backend.Properties {
|
||||
return backend.Properties{
|
||||
Connections: be.cfg.Connections,
|
||||
HasAtomicReplace: true,
|
||||
@@ -268,21 +268,26 @@ func (be *s3) Properties() backend.Properties {
|
||||
}
|
||||
|
||||
// Hasher may return a hash function for calculating a content hash for the backend
|
||||
func (be *s3) Hasher() hash.Hash {
|
||||
func (be *Backend) Hasher() hash.Hash {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Path returns the path in the bucket that is used for this backend.
|
||||
func (be *Backend) Path() string {
|
||||
return be.cfg.Prefix
|
||||
}
|
||||
|
||||
// useStorageClass returns whether file should be saved in the provided Storage Class
|
||||
// For archive storage classes, only data files are stored using that class; metadata
|
||||
// must remain instantly accessible.
|
||||
func (be *s3) useStorageClass(h backend.Handle) bool {
|
||||
func (be *Backend) useStorageClass(h backend.Handle) bool {
|
||||
isDataFile := h.Type == backend.PackFile && !h.IsMetadata
|
||||
isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass)
|
||||
return !isArchiveClass || isDataFile
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
|
||||
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
|
||||
objName := be.Filename(h)
|
||||
|
||||
opts := minio.PutObjectOptions{
|
||||
@@ -308,14 +313,14 @@ func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReade
|
||||
|
||||
// Load runs fn with a reader that yields the contents of the file at h at the
|
||||
// given offset.
|
||||
func (be *s3) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
|
||||
}
|
||||
|
||||
func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
objName := be.Filename(h)
|
||||
opts := minio.GetObjectOptions{}
|
||||
|
||||
@@ -347,7 +352,7 @@ func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offs
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
|
||||
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
|
||||
objName := be.Filename(h)
|
||||
var obj *minio.Object
|
||||
|
||||
@@ -375,7 +380,7 @@ func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo,
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
|
||||
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
|
||||
objName := be.Filename(h)
|
||||
|
||||
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
|
||||
@@ -389,7 +394,7 @@ func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
|
||||
|
||||
// List runs fn for each file in the backend which has the type t. When an
|
||||
// error occurs (or fn returns an error), List stops and returns it.
|
||||
func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
|
||||
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
|
||||
prefix, recursive := be.Basedir(t)
|
||||
|
||||
// make sure prefix ends with a slash
|
||||
@@ -444,15 +449,15 @@ func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.File
|
||||
}
|
||||
|
||||
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
|
||||
func (be *s3) Delete(ctx context.Context) error {
|
||||
func (be *Backend) Delete(ctx context.Context) error {
|
||||
return util.DefaultDelete(ctx, be)
|
||||
}
|
||||
|
||||
// Close does nothing
|
||||
func (be *s3) Close() error { return nil }
|
||||
func (be *Backend) Close() error { return nil }
|
||||
|
||||
// Warmup transitions handles from cold to hot storage if needed.
|
||||
func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
|
||||
func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
|
||||
handlesWarmingUp := []backend.Handle{}
|
||||
|
||||
if be.cfg.EnableRestore {
|
||||
@@ -473,7 +478,7 @@ func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.H
|
||||
}
|
||||
|
||||
// requestRestore sends a glacier restore request on a given file.
|
||||
func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error) {
|
||||
func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) {
|
||||
objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -509,7 +514,7 @@ func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error)
|
||||
}
|
||||
|
||||
// getWarmupStatus returns the warmup status of the provided object.
|
||||
func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
|
||||
func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
|
||||
// We can't use objectInfo.StorageClass to get the storage class of the
|
||||
// object because this field is only set during ListObjects operations.
|
||||
// The response header is the documented way to get the storage class
|
||||
@@ -540,7 +545,7 @@ func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
|
||||
}
|
||||
|
||||
// WarmupWait waits until all handles are in hot storage.
|
||||
func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
|
||||
func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error {
|
||||
timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout)
|
||||
defer timeoutCtxCancel()
|
||||
|
||||
@@ -559,7 +564,7 @@ func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
|
||||
}
|
||||
|
||||
// waitForRestore waits for a given file to be restored.
|
||||
func (be *s3) waitForRestore(ctx context.Context, filename string) error {
|
||||
func (be *Backend) waitForRestore(ctx context.Context, filename string) error {
|
||||
for {
|
||||
var objectInfo minio.ObjectInfo
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -37,7 +38,8 @@ type SFTP struct {
|
||||
cmd *exec.Cmd
|
||||
result <-chan error
|
||||
|
||||
posixRename bool
|
||||
posixRename bool
|
||||
chmodBeforeRemove atomic.Bool
|
||||
|
||||
layout.Layout
|
||||
Config
|
||||
@@ -363,6 +365,7 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
|
||||
if err == nil {
|
||||
err = f.Chmod(r.Modes.File)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return errors.Wrapf(err, "Chmod %v", tmpFilename)
|
||||
}
|
||||
}
|
||||
@@ -404,12 +407,11 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
|
||||
} else {
|
||||
err = r.c.Rename(tmpFilename, filename)
|
||||
}
|
||||
err = setFileReadonly(r.c, filename, r.Modes.File)
|
||||
if err != nil {
|
||||
return errors.Errorf("sftp setFileReadonly: %v", err)
|
||||
return errors.Wrapf(err, "Rename %v", tmpFilename)
|
||||
}
|
||||
|
||||
return errors.Wrapf(err, "Rename %v", tmpFilename)
|
||||
err = setFileReadonly(r.c, filename, r.Modes.File)
|
||||
return errors.Wrapf(err, "setFileReadonly %v", filename)
|
||||
}
|
||||
|
||||
// checkNoSpace checks if err was likely caused by lack of available space
|
||||
@@ -507,7 +509,37 @@ func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return errors.Wrapf(r.c.Remove(r.Filename(h)), "Remove %v", r.Filename(h))
|
||||
path := r.Filename(h)
|
||||
|
||||
if r.chmodBeforeRemove.Load() {
|
||||
return r.removeWithChmod(path)
|
||||
}
|
||||
|
||||
// optimistically try to remove the file
|
||||
err := r.c.Remove(path)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrPermission) {
|
||||
return errors.Wrapf(err, "Remove %v", path)
|
||||
}
|
||||
|
||||
// fallback to chmod + remove
|
||||
// this is necessary on Windows where read-only files cannot be deleted without chmod.
|
||||
if err := r.removeWithChmod(path); err != nil {
|
||||
return err
|
||||
}
|
||||
r.chmodBeforeRemove.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *SFTP) removeWithChmod(path string) error {
|
||||
err := r.c.Chmod(path, r.Modes.File)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "Chmod %v", path)
|
||||
}
|
||||
|
||||
return errors.Wrapf(r.c.Remove(path), "Remove %v", path)
|
||||
}
|
||||
|
||||
// List runs fn for each file in the backend which has the type t. When an
|
||||
|
||||
@@ -251,6 +251,10 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
|
||||
test.OK(t, b.Remove(context.TODO(), handle))
|
||||
}
|
||||
|
||||
type setter interface {
|
||||
SetListMaxItems(int)
|
||||
}
|
||||
|
||||
// TestList makes sure that the backend implements List() pagination correctly.
|
||||
func (s *Suite[C]) TestList(t *testing.T) {
|
||||
random := seedRand(t)
|
||||
@@ -298,39 +302,54 @@ func (s *Suite[C]) TestList(t *testing.T) {
|
||||
|
||||
t.Logf("wrote %v files", len(list1))
|
||||
|
||||
list2 := make(map[restic.ID]int64)
|
||||
|
||||
err = b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
|
||||
id, err := restic.ParseID(fi.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list2[id] = fi.Size
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("List returned error %v", err)
|
||||
var tests = []struct {
|
||||
maxItems int
|
||||
}{
|
||||
{11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123},
|
||||
}
|
||||
|
||||
t.Logf("loaded %v IDs from backend", len(list2))
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
|
||||
list2 := make(map[restic.ID]int64)
|
||||
|
||||
for id, size := range list1 {
|
||||
size2, ok := list2[id]
|
||||
if !ok {
|
||||
t.Errorf("id %v not returned by List()", id.Str())
|
||||
}
|
||||
if s, ok := b.(setter); ok {
|
||||
t.Logf("setting max list items to %d", test.maxItems)
|
||||
s.SetListMaxItems(test.maxItems)
|
||||
}
|
||||
|
||||
if size != size2 {
|
||||
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
|
||||
}
|
||||
}
|
||||
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
|
||||
id, err := restic.ParseID(fi.Name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list2[id] = fi.Size
|
||||
return nil
|
||||
})
|
||||
|
||||
for id := range list2 {
|
||||
_, ok := list1[id]
|
||||
if !ok {
|
||||
t.Errorf("extra id %v returned by List()", id.Str())
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("List returned error %v", err)
|
||||
}
|
||||
|
||||
t.Logf("loaded %v IDs from backend", len(list2))
|
||||
|
||||
for id, size := range list1 {
|
||||
size2, ok := list2[id]
|
||||
if !ok {
|
||||
t.Errorf("id %v not returned by List()", id.Str())
|
||||
}
|
||||
|
||||
if size != size2 {
|
||||
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
|
||||
}
|
||||
}
|
||||
|
||||
for id := range list2 {
|
||||
_, ok := list1[id]
|
||||
if !ok {
|
||||
t.Errorf("extra id %v returned by List()", id.Str())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Logf("remove %d files", numTestFiles)
|
||||
|
||||
@@ -35,10 +35,15 @@ func DefaultDelete(ctx context.Context, be backend.Backend) error {
|
||||
|
||||
for _, t := range alltypes {
|
||||
err := be.List(ctx, t, func(fi backend.FileInfo) error {
|
||||
return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
|
||||
err := be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
|
||||
if err != nil && be.IsNotExist(err) {
|
||||
// deletion of files created by TestSaveError may happen with a delay for the REST server, so we ignore the error
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
}
|
||||
err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
"github.com/restic/restic/internal/data"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/repository/hashing"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
@@ -73,8 +75,7 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) {
|
||||
}
|
||||
|
||||
func TestCheckRepo(t *testing.T) {
|
||||
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
defer cleanup()
|
||||
repo, _ := repository.TestFromFixture(t, checkerTestData)
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
@@ -91,8 +92,7 @@ func TestCheckRepo(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMissingPack(t *testing.T) {
|
||||
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
defer cleanup()
|
||||
repo, be := repository.TestFromFixture(t, checkerTestData)
|
||||
|
||||
packID := restic.TestParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6")
|
||||
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: packID.String()}))
|
||||
@@ -109,7 +109,7 @@ func TestMissingPack(t *testing.T) {
|
||||
test.Assert(t, len(errs) == 1,
|
||||
"expected exactly one error, got %v", len(errs))
|
||||
|
||||
if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
|
||||
if err, ok := errs[0].(*repository.PackError); ok {
|
||||
test.Equals(t, packID, err.ID)
|
||||
} else {
|
||||
t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
|
||||
@@ -117,8 +117,7 @@ func TestMissingPack(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestUnreferencedPack(t *testing.T) {
|
||||
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
defer cleanup()
|
||||
repo, be := repository.TestFromFixture(t, checkerTestData)
|
||||
|
||||
// index 3f1a only references pack 60e0
|
||||
packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
|
||||
@@ -137,7 +136,7 @@ func TestUnreferencedPack(t *testing.T) {
|
||||
test.Assert(t, len(errs) == 1,
|
||||
"expected exactly one error, got %v", len(errs))
|
||||
|
||||
if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
|
||||
if err, ok := errs[0].(*repository.PackError); ok {
|
||||
test.Equals(t, packID, err.ID.String())
|
||||
} else {
|
||||
t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
|
||||
@@ -145,8 +144,7 @@ func TestUnreferencedPack(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestUnreferencedBlobs(t *testing.T) {
|
||||
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
defer cleanup()
|
||||
repo, be := repository.TestFromFixture(t, checkerTestData)
|
||||
|
||||
snapshotID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
|
||||
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.SnapshotFile, Name: snapshotID.String()}))
|
||||
@@ -180,8 +178,7 @@ func TestUnreferencedBlobs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestModifiedIndex(t *testing.T) {
|
||||
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
defer cleanup()
|
||||
repo, be := repository.TestFromFixture(t, checkerTestData)
|
||||
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
@@ -190,19 +187,56 @@ func TestModifiedIndex(t *testing.T) {
|
||||
Type: restic.IndexFile,
|
||||
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
|
||||
}
|
||||
var data []byte
|
||||
test.OK(t, be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
|
||||
var err error
|
||||
data, err = io.ReadAll(rd)
|
||||
|
||||
tmpfile, err := os.CreateTemp("", "restic-test-mod-index-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
err := tmpfile.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = os.Remove(tmpfile.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
wr := io.Writer(tmpfile)
|
||||
var hw *hashing.Writer
|
||||
if be.Hasher() != nil {
|
||||
hw = hashing.NewWriter(wr, be.Hasher())
|
||||
wr = hw
|
||||
}
|
||||
|
||||
// read the file from the backend
|
||||
err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
|
||||
_, err := io.Copy(wr, rd)
|
||||
return err
|
||||
}))
|
||||
})
|
||||
test.OK(t, err)
|
||||
|
||||
// save the index again with a modified name so that the hash doesn't match
|
||||
// the content any more
|
||||
h2 := backend.Handle{
|
||||
Type: restic.IndexFile,
|
||||
Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
|
||||
}
|
||||
test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher())))
|
||||
|
||||
var hash []byte
|
||||
if hw != nil {
|
||||
hash = hw.Sum(nil)
|
||||
}
|
||||
rd, err := backend.NewFileReader(tmpfile, hash)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = be.Save(context.TODO(), h2, rd)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
@@ -220,8 +254,7 @@ func TestModifiedIndex(t *testing.T) {
|
||||
var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
|
||||
|
||||
func TestDuplicatePacksInIndex(t *testing.T) {
|
||||
repo, _, cleanup := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
|
||||
defer cleanup()
|
||||
repo, _ := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
@@ -420,8 +453,7 @@ func (r *loadTreesOnceRepository) LoadBlob(ctx context.Context, t restic.BlobTyp
|
||||
}
|
||||
|
||||
func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
|
||||
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
defer cleanup()
|
||||
repo, _ := repository.TestFromFixture(t, checkerTestData)
|
||||
checkRepo := &loadTreesOnceRepository{
|
||||
Repository: repo,
|
||||
loadedTrees: restic.NewIDSet(),
|
||||
@@ -569,13 +601,12 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
|
||||
test.Assert(t, delayRepo.Triggered, "delay repository did not trigger")
|
||||
}
|
||||
|
||||
func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, func()) {
|
||||
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
|
||||
func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository) {
|
||||
repo, _ := repository.TestFromFixture(t, checkerTestData)
|
||||
|
||||
chkr := checker.New(repo, false)
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
if len(errs) > 0 {
|
||||
defer cleanup()
|
||||
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
|
||||
}
|
||||
|
||||
@@ -584,12 +615,11 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun
|
||||
t.Fatalf("expected mixed pack hint, got %v", err)
|
||||
}
|
||||
}
|
||||
return chkr, repo, cleanup
|
||||
return chkr, repo
|
||||
}
|
||||
|
||||
func BenchmarkChecker(t *testing.B) {
|
||||
chkr, _, cleanup := loadBenchRepository(t)
|
||||
defer cleanup()
|
||||
chkr, _ := loadBenchRepository(t)
|
||||
|
||||
t.ResetTimer()
|
||||
|
||||
@@ -601,8 +631,7 @@ func BenchmarkChecker(t *testing.B) {
|
||||
}
|
||||
|
||||
func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) {
|
||||
chkr, repo, cleanup := loadBenchRepository(t)
|
||||
defer cleanup()
|
||||
chkr, repo := loadBenchRepository(t)
|
||||
|
||||
snID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
|
||||
sn2, err := data.LoadSnapshot(context.TODO(), repo, snID)
|
||||
|
||||
@@ -109,7 +109,7 @@ func nodeFillExtendedAttributes(node *data.Node, path string, ignoreListError bo
|
||||
for _, attr := range xattrs {
|
||||
attrVal, err := getxattr(path, attr)
|
||||
if err != nil {
|
||||
warnf("can not obtain extended attribute %v for %v:\n", attr, path)
|
||||
warnf("can not obtain extended attribute %v for %v: %v\n", attr, path, err)
|
||||
continue
|
||||
}
|
||||
attr := data.ExtendedAttribute{
|
||||
|
||||
@@ -225,7 +225,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
|
||||
return d.cache.lookupOrCreate(name, -1, func(forget forgetFn) (fs.Node, error) {
|
||||
node, ok := d.items[name]
|
||||
if !ok {
|
||||
debug.Log(" Lookup(%v) -> not found", name)
|
||||
|
||||
@@ -247,6 +247,39 @@ func TestStableNodeObjects(t *testing.T) {
|
||||
testStableLookup(t, dir, "file-2")
|
||||
}
|
||||
|
||||
func TestSnapshotsDirLatestSymlinkUpdatesAfterReload(t *testing.T) {
|
||||
repo := repository.TestRepository(t)
|
||||
timeTemplate := "2006-01-02T15:04:05"
|
||||
|
||||
firstTime, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56Z")
|
||||
rtest.OK(t, err)
|
||||
secondTime := firstTime.Add(time.Hour)
|
||||
|
||||
data.TestCreateSnapshot(t, repo, firstTime, 0)
|
||||
root := NewRoot(repo, Config{
|
||||
TimeTemplate: timeTemplate,
|
||||
PathTemplates: []string{"snapshots/%T"},
|
||||
})
|
||||
|
||||
snapshotsDir := testStableLookup(t, root, "snapshots")
|
||||
rtest.Equals(t, firstTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
|
||||
|
||||
data.TestCreateSnapshot(t, repo, secondTime, 0)
|
||||
root.SnapshotsDir.dirStruct.lastCheck = time.Now().Add(-2 * minSnapshotsReloadTime)
|
||||
|
||||
rtest.Equals(t, secondTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
|
||||
}
|
||||
|
||||
func readLatestTarget(t testing.TB, node fs.Node) string {
|
||||
t.Helper()
|
||||
|
||||
latest, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), "latest")
|
||||
rtest.OK(t, err)
|
||||
target, err := latest.(fs.NodeReadlinker).Readlink(context.TODO(), nil)
|
||||
rtest.OK(t, err)
|
||||
return target
|
||||
}
|
||||
|
||||
// Test reporting of fuse.Attr.Blocks in multiples of 512.
|
||||
func TestBlocks(t *testing.T) {
|
||||
root := &Root{}
|
||||
|
||||
@@ -61,7 +61,7 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
|
||||
debug.Log("ReadDirAll()")
|
||||
|
||||
// update snapshots
|
||||
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
|
||||
meta, _, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
|
||||
if err != nil {
|
||||
return nil, unwrapCtxCanceled(err)
|
||||
} else if meta == nil {
|
||||
@@ -104,14 +104,14 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
|
||||
func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
|
||||
debug.Log("Lookup(%s)", name)
|
||||
|
||||
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
|
||||
meta, gen, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
|
||||
if err != nil {
|
||||
return nil, unwrapCtxCanceled(err)
|
||||
} else if meta == nil {
|
||||
return nil, syscall.ENOENT
|
||||
}
|
||||
|
||||
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
|
||||
return d.cache.lookupOrCreate(name, gen, func(forget forgetFn) (fs.Node, error) {
|
||||
entry := meta.names[name]
|
||||
if entry == nil {
|
||||
return nil, syscall.ENOENT
|
||||
|
||||
@@ -42,6 +42,10 @@ type SnapshotsDirStructure struct {
|
||||
|
||||
hash [sha256.Size]byte // Hash at last check.
|
||||
lastCheck time.Time
|
||||
|
||||
// generation is incremented whenever the directory structure is rebuilt.
|
||||
// It allows treeCache instances to detect stale entries and reset themselves.
|
||||
generation int64
|
||||
}
|
||||
|
||||
// NewSnapshotsDirStructure returns a new directory structure for snapshots.
|
||||
@@ -280,6 +284,7 @@ func (d *SnapshotsDirStructure) makeDirs(snapshots data.Snapshots) {
|
||||
}
|
||||
|
||||
d.entries = entries
|
||||
d.generation++
|
||||
}
|
||||
|
||||
const minSnapshotsReloadTime = 60 * time.Second
|
||||
@@ -337,13 +342,13 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) {
|
||||
func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, int64, error) {
|
||||
err := d.updateSnapshots(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
d.mutex.Lock()
|
||||
defer d.mutex.Unlock()
|
||||
return d.entries[prefix], nil
|
||||
return d.entries[prefix], d.generation, nil
|
||||
}
|
||||
|
||||
@@ -5,12 +5,15 @@ package fuse
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
|
||||
"github.com/anacrolix/fuse/fs"
|
||||
)
|
||||
|
||||
type treeCache struct {
|
||||
nodes map[string]fs.Node
|
||||
m sync.Mutex
|
||||
nodes map[string]fs.Node
|
||||
m sync.Mutex
|
||||
generation int64
|
||||
}
|
||||
|
||||
type forgetFn func()
|
||||
@@ -21,19 +24,28 @@ func newTreeCache() *treeCache {
|
||||
}
|
||||
}
|
||||
|
||||
func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
|
||||
func (t *treeCache) lookupOrCreate(name string, generation int64, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
|
||||
t.m.Lock()
|
||||
defer t.m.Unlock()
|
||||
|
||||
if generation >= 0 && generation != t.generation {
|
||||
debug.Log("treeCache generation changed %d -> %d, resetting cache", t.generation, generation)
|
||||
t.nodes = make(map[string]fs.Node)
|
||||
t.generation = generation
|
||||
}
|
||||
|
||||
if node, ok := t.nodes[name]; ok {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
cacheGeneration := t.generation
|
||||
node, err := create(func() {
|
||||
t.m.Lock()
|
||||
defer t.m.Unlock()
|
||||
|
||||
delete(t.nodes, name)
|
||||
if t.generation == cacheGeneration {
|
||||
delete(t.nodes, name)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build darwin || freebsd || linux
|
||||
|
||||
package fuse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/anacrolix/fuse"
|
||||
"github.com/anacrolix/fuse/fs"
|
||||
|
||||
"github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
type cacheTestNode struct {
|
||||
id int
|
||||
}
|
||||
|
||||
func (n cacheTestNode) Attr(context.Context, *fuse.Attr) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTreeCacheGeneration(t *testing.T) {
|
||||
cache := newTreeCache()
|
||||
created := 0
|
||||
create := func(forgetFn) (fs.Node, error) {
|
||||
created++
|
||||
return cacheTestNode{id: created}, nil
|
||||
}
|
||||
|
||||
node1, err := cache.lookupOrCreate("node", 1, create)
|
||||
test.OK(t, err)
|
||||
node2, err := cache.lookupOrCreate("node", 1, create)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, node1 == node2, "lookup should reuse cached node")
|
||||
test.Equals(t, 1, created)
|
||||
|
||||
node3, err := cache.lookupOrCreate("node", 2, create)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, node1 != node3, "lookup should recreate node after generation change")
|
||||
test.Equals(t, 2, created)
|
||||
|
||||
node4, err := cache.lookupOrCreate("node", -1, create)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, node3 == node4, "negative generation should not reset cache")
|
||||
test.Equals(t, 2, created)
|
||||
|
||||
node5, err := cache.lookupOrCreate("node", 3, create)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, node4 != node5, "lookup should still track later generation changes")
|
||||
test.Equals(t, 3, created)
|
||||
}
|
||||
|
||||
func TestTreeCacheForgetOnlyRemovesSameGeneration(t *testing.T) {
|
||||
cache := newTreeCache()
|
||||
created := 0
|
||||
var forgets []forgetFn
|
||||
create := func(forget forgetFn) (fs.Node, error) {
|
||||
created++
|
||||
forgets = append(forgets, forget)
|
||||
return cacheTestNode{id: created}, nil
|
||||
}
|
||||
|
||||
node1, err := cache.lookupOrCreate("node", 1, create)
|
||||
test.OK(t, err)
|
||||
node2, err := cache.lookupOrCreate("node", 2, create)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, node1 != node2, "lookup should recreate node after generation change")
|
||||
test.Equals(t, 2, created)
|
||||
|
||||
forgets[0]()
|
||||
node3, err := cache.lookupOrCreate("node", 2, create)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, node2 == node3, "forget from an old generation must not remove the current node")
|
||||
test.Equals(t, 2, created)
|
||||
|
||||
forgets[1]()
|
||||
node4, err := cache.lookupOrCreate("node", 2, create)
|
||||
test.OK(t, err)
|
||||
test.Assert(t, node3 != node4, "forget from the current generation should remove the cached node")
|
||||
test.Equals(t, 3, created)
|
||||
}
|
||||
@@ -35,7 +35,7 @@ import (
|
||||
// to a missing backend storage location or config file
|
||||
var ErrNoRepository = errors.New("repository does not exist")
|
||||
|
||||
const Version = "0.19.0-dev (compiled manually)"
|
||||
const Version = "0.19.1"
|
||||
|
||||
// TimeFormat is the format used for all timestamps printed by restic.
|
||||
const TimeFormat = "2006-01-02 15:04:05"
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/repository/hashing"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// ErrPackData is returned if errors are discovered while verifying a packfile
|
||||
type ErrPackData struct {
|
||||
PackID restic.ID
|
||||
errs []error
|
||||
}
|
||||
|
||||
func (e *ErrPackData) Error() string {
|
||||
return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
|
||||
}
|
||||
|
||||
type partialReadError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *partialReadError) Error() string {
|
||||
return e.err.Error()
|
||||
}
|
||||
|
||||
// CheckPack reads a pack and checks the integrity of all blobs.
|
||||
func CheckPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
|
||||
if err != nil {
|
||||
if r.cache != nil {
|
||||
// ignore error as there's not much we can do here
|
||||
_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
|
||||
}
|
||||
|
||||
// retry pack verification to detect transient errors
|
||||
err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
|
||||
if err2 != nil {
|
||||
err = err2
|
||||
} else {
|
||||
err = fmt.Errorf("check successful on second attempt, original error %w", err)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
|
||||
debug.Log("checking pack %v", id.String())
|
||||
|
||||
if len(blobs) == 0 {
|
||||
return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
|
||||
}
|
||||
|
||||
// sanity check blobs in index
|
||||
blobs.Sort()
|
||||
idxHdrSize := pack.CalculateHeaderSize(blobs)
|
||||
lastBlobEnd := 0
|
||||
nonContinuousPack := false
|
||||
for _, blob := range blobs {
|
||||
if lastBlobEnd != int(blob.Offset) {
|
||||
nonContinuousPack = true
|
||||
}
|
||||
lastBlobEnd = int(blob.Offset + blob.Length)
|
||||
}
|
||||
// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
|
||||
|
||||
var errs []error
|
||||
if nonContinuousPack {
|
||||
debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
|
||||
errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
|
||||
}
|
||||
|
||||
// calculate hash on-the-fly while reading the pack and capture pack header
|
||||
var hash restic.ID
|
||||
var hdrBuf []byte
|
||||
// must use a separate slice from `errs` here as we're only interested in the last retry
|
||||
var blobErrors []error
|
||||
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
|
||||
err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
|
||||
hrd := hashing.NewReader(rd, sha256.New())
|
||||
bufRd.Reset(hrd)
|
||||
// reset blob errors for each retry
|
||||
blobErrors = nil
|
||||
|
||||
it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
val, err := it.Next()
|
||||
if err == errPackEOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return &partialReadError{err}
|
||||
}
|
||||
debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle)
|
||||
if val.Err != nil {
|
||||
debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err)
|
||||
blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
|
||||
}
|
||||
}
|
||||
|
||||
// skip enough bytes until we reach the possible header start
|
||||
curPos := lastBlobEnd
|
||||
minHdrStart := int(size) - pack.MaxHeaderSize
|
||||
if minHdrStart > curPos {
|
||||
_, err := bufRd.Discard(minHdrStart - curPos)
|
||||
if err != nil {
|
||||
return &partialReadError{err}
|
||||
}
|
||||
curPos += minHdrStart - curPos
|
||||
}
|
||||
|
||||
// read remainder, which should be the pack header
|
||||
var err error
|
||||
hdrBuf = make([]byte, int(size-int64(curPos)))
|
||||
_, err = io.ReadFull(bufRd, hdrBuf)
|
||||
if err != nil {
|
||||
return &partialReadError{err}
|
||||
}
|
||||
|
||||
hash = restic.IDFromHash(hrd.Sum(nil))
|
||||
return nil
|
||||
})
|
||||
errs = append(errs, blobErrors...)
|
||||
if err != nil {
|
||||
var e *partialReadError
|
||||
isPartialReadError := errors.As(err, &e)
|
||||
// failed to load the pack file, return as further checks cannot succeed anyways
|
||||
debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err)
|
||||
if isPartialReadError {
|
||||
return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
|
||||
}
|
||||
|
||||
// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
|
||||
// completely failed to download such that there's no point in repairing anything.
|
||||
return fmt.Errorf("download error: %w", err)
|
||||
}
|
||||
if !hash.Equal(id) {
|
||||
debug.Log("pack ID does not match, want %v, got %v", id, hash)
|
||||
return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
|
||||
}
|
||||
|
||||
blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
|
||||
if err != nil {
|
||||
return &ErrPackData{PackID: id, errs: append(errs, err)}
|
||||
}
|
||||
|
||||
if uint32(idxHdrSize) != hdrSize {
|
||||
debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
|
||||
errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
|
||||
}
|
||||
|
||||
for _, blob := range blobs {
|
||||
// Check if blob is contained in index and position is correct
|
||||
idxHas := false
|
||||
for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
|
||||
if pb.PackID == id && pb.Blob == blob {
|
||||
idxHas = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !idxHas {
|
||||
errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return &ErrPackData{PackID: id, errs: errs}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type bufReader struct {
|
||||
rd *bufio.Reader
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func newBufReader(rd *bufio.Reader) *bufReader {
|
||||
return &bufReader{
|
||||
rd: rd,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bufReader) Discard(n int) (discarded int, err error) {
|
||||
return b.rd.Discard(n)
|
||||
}
|
||||
|
||||
func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
|
||||
if cap(b.buf) < n {
|
||||
b.buf = make([]byte, n)
|
||||
}
|
||||
b.buf = b.buf[:n]
|
||||
|
||||
_, err = io.ReadFull(b.rd, b.buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.buf, nil
|
||||
}
|
||||
@@ -2,17 +2,12 @@ package repository
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/repository/hashing"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
@@ -51,29 +46,18 @@ func (e *ErrMixedPack) Error() string {
|
||||
return fmt.Sprintf("pack %v contains a mix of tree and data blobs", e.PackID.Str())
|
||||
}
|
||||
|
||||
// ErrPackMetadata describes an error with a specific pack. It is used for missing, truncated or orphaned packs.
|
||||
// Errors of the actual pack data are returned as ErrPackData.
|
||||
type ErrPackMetadata struct {
|
||||
// PackError describes an error with a specific pack.
|
||||
type PackError struct {
|
||||
ID restic.ID
|
||||
Orphaned bool
|
||||
Truncated bool
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *ErrPackMetadata) Error() string {
|
||||
func (e *PackError) Error() string {
|
||||
return "pack " + e.ID.String() + ": " + e.Err.Error()
|
||||
}
|
||||
|
||||
// ErrPackData is returned if errors are discovered while verifying a packfile
|
||||
type ErrPackData struct {
|
||||
PackID restic.ID
|
||||
errs []error
|
||||
}
|
||||
|
||||
func (e *ErrPackData) Error() string {
|
||||
return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
|
||||
}
|
||||
|
||||
// Checker handles index-related operations for repository checking.
|
||||
type Checker struct {
|
||||
repo *Repository
|
||||
@@ -215,7 +199,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case errChan <- &ErrPackMetadata{ID: id, Err: errors.New("does not exist")}:
|
||||
case errChan <- &PackError{ID: id, Err: errors.New("does not exist")}:
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -225,7 +209,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case errChan <- &ErrPackMetadata{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
|
||||
case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -235,7 +219,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case errChan <- &ErrPackMetadata{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
|
||||
case errChan <- &PackError{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -269,7 +253,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
|
||||
bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize)
|
||||
dec, err := zstd.NewReader(nil)
|
||||
if err != nil {
|
||||
panic(dec)
|
||||
panic(err)
|
||||
}
|
||||
defer dec.Close()
|
||||
for {
|
||||
@@ -285,7 +269,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
|
||||
}
|
||||
}
|
||||
|
||||
err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
|
||||
err := CheckPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
|
||||
p.Add(1)
|
||||
if err == nil {
|
||||
continue
|
||||
@@ -325,186 +309,3 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkPack reads a pack and checks the integrity of all blobs.
|
||||
func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
|
||||
if err != nil {
|
||||
if r.cache != nil {
|
||||
// ignore error as there's not much we can do here
|
||||
_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
|
||||
}
|
||||
|
||||
// retry pack verification to detect transient errors
|
||||
err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
|
||||
if err2 != nil {
|
||||
err = err2
|
||||
} else {
|
||||
err = fmt.Errorf("check successful on second attempt, original error %w", err)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
|
||||
|
||||
type partialReadError struct {
|
||||
error
|
||||
}
|
||||
|
||||
debug.Log("checking pack %v", id.String())
|
||||
|
||||
if len(blobs) == 0 {
|
||||
return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
|
||||
}
|
||||
|
||||
// sanity check blobs in index
|
||||
blobs.Sort()
|
||||
idxHdrSize := pack.CalculateHeaderSize(blobs)
|
||||
lastBlobEnd := 0
|
||||
nonContinuousPack := false
|
||||
for _, blob := range blobs {
|
||||
if lastBlobEnd != int(blob.Offset) {
|
||||
nonContinuousPack = true
|
||||
}
|
||||
lastBlobEnd = int(blob.Offset + blob.Length)
|
||||
}
|
||||
// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
|
||||
|
||||
var errs []error
|
||||
if nonContinuousPack {
|
||||
debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
|
||||
errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
|
||||
}
|
||||
|
||||
// calculate hash on-the-fly while reading the pack and capture pack header
|
||||
var hash restic.ID
|
||||
var hdrBuf []byte
|
||||
// must use a separate slice from `errs` here as we're only interested in the last retry
|
||||
var blobErrors []error
|
||||
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
|
||||
err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
|
||||
hrd := hashing.NewReader(rd, sha256.New())
|
||||
bufRd.Reset(hrd)
|
||||
// reset blob errors for each retry
|
||||
blobErrors = nil
|
||||
|
||||
it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
val, err := it.Next()
|
||||
if err == errPackEOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return &partialReadError{err}
|
||||
}
|
||||
debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle)
|
||||
if val.Err != nil {
|
||||
debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err)
|
||||
blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
|
||||
}
|
||||
}
|
||||
|
||||
// skip enough bytes until we reach the possible header start
|
||||
curPos := lastBlobEnd
|
||||
minHdrStart := int(size) - pack.MaxHeaderSize
|
||||
if minHdrStart > curPos {
|
||||
_, err := bufRd.Discard(minHdrStart - curPos)
|
||||
if err != nil {
|
||||
return &partialReadError{err}
|
||||
}
|
||||
curPos += minHdrStart - curPos
|
||||
}
|
||||
|
||||
// read remainder, which should be the pack header
|
||||
var err error
|
||||
hdrBuf = make([]byte, int(size-int64(curPos)))
|
||||
_, err = io.ReadFull(bufRd, hdrBuf)
|
||||
if err != nil {
|
||||
return &partialReadError{err}
|
||||
}
|
||||
|
||||
hash = restic.IDFromHash(hrd.Sum(nil))
|
||||
return nil
|
||||
})
|
||||
errs = append(errs, blobErrors...)
|
||||
if err != nil {
|
||||
var e *partialReadError
|
||||
isPartialReadError := errors.As(err, &e)
|
||||
// failed to load the pack file, return as further checks cannot succeed anyways
|
||||
debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err)
|
||||
if isPartialReadError {
|
||||
return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
|
||||
}
|
||||
|
||||
// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
|
||||
// completely failed to download such that there's no point in repairing anything.
|
||||
return fmt.Errorf("download error: %w", err)
|
||||
}
|
||||
if !hash.Equal(id) {
|
||||
debug.Log("pack ID does not match, want %v, got %v", id, hash)
|
||||
return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
|
||||
}
|
||||
|
||||
blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
|
||||
if err != nil {
|
||||
return &ErrPackData{PackID: id, errs: append(errs, err)}
|
||||
}
|
||||
|
||||
if uint32(idxHdrSize) != hdrSize {
|
||||
debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
|
||||
errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
|
||||
}
|
||||
|
||||
for _, blob := range blobs {
|
||||
// Check if blob is contained in index and position is correct
|
||||
idxHas := false
|
||||
for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
|
||||
if pb.PackID == id && pb.Blob == blob {
|
||||
idxHas = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !idxHas {
|
||||
errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return &ErrPackData{PackID: id, errs: errs}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type bufReader struct {
|
||||
rd *bufio.Reader
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func newBufReader(rd *bufio.Reader) *bufReader {
|
||||
return &bufReader{
|
||||
rd: rd,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bufReader) Discard(n int) (discarded int, err error) {
|
||||
return b.rd.Discard(n)
|
||||
}
|
||||
|
||||
func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
|
||||
if cap(b.buf) < n {
|
||||
b.buf = make([]byte, n)
|
||||
}
|
||||
b.buf = b.buf[:n]
|
||||
|
||||
_, err = io.ReadFull(b.rd, b.buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.buf, nil
|
||||
}
|
||||
|
||||
@@ -1,390 +0,0 @@
|
||||
//go:build debug
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/restic/restic/internal/crypto"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/repository/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
type packDumpEntry struct {
|
||||
Name string `json:"name"`
|
||||
Blobs []packDumpBlob `json:"blobs"`
|
||||
}
|
||||
|
||||
type packDumpBlob struct {
|
||||
Type restic.BlobType `json:"type"`
|
||||
Length uint `json:"length"`
|
||||
ID restic.ID `json:"id"`
|
||||
Offset uint `json:"offset"`
|
||||
}
|
||||
|
||||
func writePackDumpJSON(wr io.Writer, item any) error {
|
||||
buf, err := json.MarshalIndent(item, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = wr.Write(append(buf, '\n'))
|
||||
return err
|
||||
}
|
||||
|
||||
// DumpPacks lists each pack file and writes its header blob layout as JSON to wr.
|
||||
func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
|
||||
var m sync.Mutex
|
||||
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
|
||||
blobs, err := repo.ListPack(ctx, id, size)
|
||||
if err != nil {
|
||||
printer.E("error for pack %v: %v", id.Str(), err)
|
||||
return nil
|
||||
}
|
||||
|
||||
p := packDumpEntry{
|
||||
Name: id.String(),
|
||||
Blobs: make([]packDumpBlob, len(blobs)),
|
||||
}
|
||||
for i, blob := range blobs {
|
||||
p.Blobs[i] = packDumpBlob{
|
||||
Type: blob.Type,
|
||||
Length: blob.Length,
|
||||
ID: blob.ID,
|
||||
Offset: blob.Offset,
|
||||
}
|
||||
}
|
||||
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
return writePackDumpJSON(wr, p)
|
||||
})
|
||||
}
|
||||
|
||||
// DumpIndexes loads each on-disk index file and writes its debug dump to wr.
|
||||
func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
|
||||
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
|
||||
printer.S("index_id: %v", id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return idx.Dump(wr)
|
||||
})
|
||||
}
|
||||
|
||||
// ExaminePackOptions configures debug examination of a pack file.
|
||||
type ExaminePackOptions struct {
|
||||
TryRepair bool
|
||||
RepairByte bool
|
||||
ExtractPack bool
|
||||
ReuploadBlobs bool
|
||||
}
|
||||
|
||||
// ExaminePack loads and inspects a pack file and its index entries.
|
||||
func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error {
|
||||
printer.S("examine %v", id)
|
||||
|
||||
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
|
||||
// also process damaged pack files
|
||||
if buf == nil {
|
||||
return err
|
||||
}
|
||||
printer.S(" file size is %v", len(buf))
|
||||
gotID := restic.Hash(buf)
|
||||
if !id.Equal(gotID) {
|
||||
printer.S(" wanted hash %v, got %v", id, gotID)
|
||||
} else {
|
||||
printer.S(" hash for file content matches")
|
||||
}
|
||||
|
||||
printer.S(" ========================================")
|
||||
printer.S(" looking for info in the indexes")
|
||||
|
||||
blobsLoaded := false
|
||||
// examine all data the indexes have for the pack file
|
||||
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
|
||||
blobs := b.Blobs
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
checkPackSize(blobs, len(buf), printer)
|
||||
|
||||
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
|
||||
if err != nil {
|
||||
printer.E("error: %v", err)
|
||||
} else {
|
||||
blobsLoaded = true
|
||||
}
|
||||
}
|
||||
|
||||
printer.S(" ========================================")
|
||||
printer.S(" inspect the pack itself")
|
||||
|
||||
blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("pack %v: %v", id.Str(), err)
|
||||
}
|
||||
checkPackSize(blobs, len(buf), printer)
|
||||
|
||||
if !blobsLoaded {
|
||||
return loadBlobs(ctx, opts, repo, id, blobs, printer)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
|
||||
// track current size and offset
|
||||
var size, offset uint64
|
||||
|
||||
blobs.Sort()
|
||||
|
||||
for _, pb := range blobs {
|
||||
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
|
||||
if offset != uint64(pb.Offset) {
|
||||
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
|
||||
}
|
||||
offset = uint64(pb.Offset + pb.Length)
|
||||
size += uint64(pb.Length)
|
||||
}
|
||||
size += uint64(pack.CalculateHeaderSize(blobs))
|
||||
|
||||
if uint64(fileSize) != size {
|
||||
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
|
||||
} else {
|
||||
printer.S(" file sizes match")
|
||||
}
|
||||
}
|
||||
|
||||
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
|
||||
if bytewise {
|
||||
printer.S(" trying to repair blob by finding a broken byte")
|
||||
} else {
|
||||
printer.S(" trying to repair blob with single bit flip")
|
||||
}
|
||||
|
||||
ch := make(chan int)
|
||||
var wg errgroup.Group
|
||||
done := make(chan struct{})
|
||||
var fixed []byte
|
||||
var found bool
|
||||
|
||||
workers := runtime.GOMAXPROCS(0)
|
||||
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Go(func() error {
|
||||
// make a local copy of the buffer
|
||||
buf := make([]byte, len(input))
|
||||
copy(buf, input)
|
||||
|
||||
testFlip := func(idx int, pattern byte) bool {
|
||||
// flip bits
|
||||
buf[idx] ^= pattern
|
||||
|
||||
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
||||
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
|
||||
if err == nil {
|
||||
printer.S("")
|
||||
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
|
||||
printer.S(" hash is %v", restic.Hash(plaintext))
|
||||
close(done)
|
||||
found = true
|
||||
fixed = plaintext
|
||||
return true
|
||||
}
|
||||
|
||||
// flip bits back
|
||||
buf[idx] ^= pattern
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range ch {
|
||||
if bytewise {
|
||||
for j := 0; j < 255; j++ {
|
||||
if testFlip(i, byte(j)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for j := 0; j < 7; j++ {
|
||||
// flip each bit once
|
||||
if testFlip(i, (1 << uint(j))) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
wg.Go(func() error {
|
||||
defer close(ch)
|
||||
|
||||
start := time.Now()
|
||||
info := time.Now()
|
||||
for i := range input {
|
||||
select {
|
||||
case ch <- i:
|
||||
case <-done:
|
||||
printer.S(" done after %v", time.Since(start))
|
||||
return nil
|
||||
}
|
||||
|
||||
if time.Since(info) > time.Second {
|
||||
secs := time.Since(start).Seconds()
|
||||
gps := float64(i) / secs
|
||||
remaining := len(input) - i
|
||||
eta := time.Duration(float64(remaining)/gps) * time.Second
|
||||
|
||||
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
|
||||
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
|
||||
info = time.Now()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
err := wg.Wait()
|
||||
if err != nil {
|
||||
panic("all go routines can only return nil")
|
||||
}
|
||||
|
||||
if !found {
|
||||
printer.S("\n blob could not be repaired")
|
||||
}
|
||||
return fixed
|
||||
}
|
||||
|
||||
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
|
||||
// strip signature at the end
|
||||
l := len(buf)
|
||||
nonce, ct := buf[:16], buf[16:l-16]
|
||||
out := make([]byte, len(ct))
|
||||
|
||||
c, err := aes.NewCipher(k.EncryptionKey[:])
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("unable to create cipher: %v", err))
|
||||
}
|
||||
e := cipher.NewCTR(c, nonce)
|
||||
e.XORKeyStream(out, ct)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
|
||||
dec, err := zstd.NewReader(nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
packData, err := repo.LoadRaw(ctx, restic.PackFile, packID)
|
||||
// allow processing broken pack files
|
||||
if packData == nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
for _, blob := range list {
|
||||
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
|
||||
if int(blob.Offset+blob.Length) > len(packData) {
|
||||
printer.E("skipping truncated blob")
|
||||
continue
|
||||
}
|
||||
buf := packData[blob.Offset : blob.Offset+blob.Length]
|
||||
key := repo.Key()
|
||||
|
||||
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
||||
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
|
||||
outputPrefix := ""
|
||||
filePrefix := ""
|
||||
if err != nil {
|
||||
printer.E("error decrypting blob: %v", err)
|
||||
if opts.TryRepair || opts.RepairByte {
|
||||
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
|
||||
}
|
||||
if plaintext != nil {
|
||||
outputPrefix = "repaired "
|
||||
filePrefix = "repaired-"
|
||||
} else {
|
||||
plaintext = decryptUnsigned(key, buf)
|
||||
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if blob.IsCompressed() {
|
||||
decompressed, err := dec.DecodeAll(plaintext, nil)
|
||||
if err != nil {
|
||||
printer.S(" failed to decompress blob %v", blob.ID)
|
||||
}
|
||||
if decompressed != nil {
|
||||
plaintext = decompressed
|
||||
}
|
||||
}
|
||||
|
||||
id := restic.Hash(plaintext)
|
||||
var prefix string
|
||||
if !id.Equal(blob.ID) {
|
||||
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
|
||||
prefix = "wrong-hash-"
|
||||
} else {
|
||||
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
|
||||
prefix = "correct-"
|
||||
}
|
||||
if opts.ExtractPack {
|
||||
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if opts.ReuploadBlobs {
|
||||
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printer.S(" uploaded %v %v", blob.Type, id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
|
||||
filename := fmt.Sprintf("%s%s.bin", prefix, id)
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = f.Write(plain)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printer.S("decrypt of blob %v stored at %v", id, filename)
|
||||
return nil
|
||||
}
|
||||
@@ -15,8 +15,7 @@ import (
|
||||
var repoFixture = filepath.Join("..", "testdata", "test-repo.tar.gz")
|
||||
|
||||
func TestRepositoryForAllIndexes(t *testing.T) {
|
||||
repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
|
||||
defer cleanup()
|
||||
repo, _ := repository.TestFromFixture(t, repoFixture)
|
||||
|
||||
expectedIndexIDs := restic.NewIDSet()
|
||||
rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
|
||||
|
||||
@@ -280,6 +280,7 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
|
||||
}
|
||||
|
||||
func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
|
||||
defer p.Done()
|
||||
indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -302,7 +303,6 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
|
||||
return err
|
||||
}
|
||||
p.SetMax(numIndexFiles)
|
||||
defer p.Done()
|
||||
}
|
||||
|
||||
err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error {
|
||||
|
||||
@@ -694,62 +694,3 @@ func TestRewriteSplitPacks(t *testing.T) {
|
||||
blobs := mi.Lookup(blobOther.BlobHandle)
|
||||
rtest.Equals(t, nil, blobs)
|
||||
}
|
||||
|
||||
// TestRewriteFullPacks checks that Rewrite drops a duplicate full index for the same
|
||||
// pack while keeping the other index files and blob lookups intact. Creates 3 indexes:
|
||||
// - indexA: contains packA
|
||||
// - indexB: contains packB
|
||||
// - indexC: contains packB
|
||||
// After the rewrite, indexC must be dropped. The other indexes must be kept.
|
||||
func TestRewriteFullPacks(t *testing.T) {
|
||||
originalFull := index.Full
|
||||
defer func() {
|
||||
index.Full = originalFull
|
||||
}()
|
||||
index.Full = func(*index.Index) bool { return true }
|
||||
|
||||
repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion)
|
||||
|
||||
packA := restic.NewRandomID()
|
||||
packB := restic.NewRandomID()
|
||||
|
||||
blobA := restic.PackedBlob{
|
||||
PackID: packA,
|
||||
Blob: restic.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Length: uint(crypto.CiphertextLength(10)),
|
||||
Offset: 0,
|
||||
},
|
||||
}
|
||||
blobB := restic.PackedBlob{
|
||||
PackID: packB,
|
||||
Blob: restic.Blob{
|
||||
BlobHandle: restic.NewRandomBlobHandle(),
|
||||
Length: uint(crypto.CiphertextLength(50)),
|
||||
Offset: 0,
|
||||
},
|
||||
}
|
||||
|
||||
mi := index.NewMasterIndex()
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
|
||||
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
|
||||
|
||||
indexIDs := mi.IDs()
|
||||
rtest.Equals(t, 3, len(indexIDs))
|
||||
|
||||
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
|
||||
|
||||
mi2 := index.NewMasterIndex()
|
||||
rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
|
||||
|
||||
afterRewrite := mi2.IDs()
|
||||
rtest.Equals(t, 2, len(afterRewrite))
|
||||
rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
|
||||
|
||||
rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
|
||||
rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
|
||||
}
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"iter"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/repository/index"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// IndexBlob is one blob handle from an on-disk index file, or an error from loading/decoding
|
||||
// that file.
|
||||
type IndexBlob struct {
|
||||
Handle restic.BlobHandle
|
||||
Error error
|
||||
}
|
||||
|
||||
// AllIndexBlobs streams blob handles from each index file without building a master index.
|
||||
func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.LoaderUnpacked) iter.Seq[IndexBlob] {
|
||||
return func(yield func(IndexBlob) bool) {
|
||||
stopIteration := errors.New("stop index blob iteration")
|
||||
err := index.ForAllIndexes(ctx, lister, loader, func(_ restic.ID, idx *index.Index, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for blob := range idx.Values() {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if !yield(IndexBlob{Handle: blob.BlobHandle}) {
|
||||
return stopIteration
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil && !errors.Is(err, stopIteration) {
|
||||
yield(IndexBlob{Error: err})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
package repository_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func TestAllIndexBlobs(t *testing.T) {
|
||||
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
|
||||
|
||||
want := restic.NewBlobSet()
|
||||
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
for i := range 5 {
|
||||
data := []byte{byte('a' + i)}
|
||||
id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, data, restic.ID{}, false)
|
||||
rtest.OK(t, err)
|
||||
want.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id})
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
|
||||
fromMaster := restic.NewBlobSet()
|
||||
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
|
||||
fromMaster.Insert(pb.BlobHandle)
|
||||
}))
|
||||
rtest.Equals(t, want, fromMaster)
|
||||
|
||||
fromStream := restic.NewBlobSet()
|
||||
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
|
||||
if entry.Error != nil {
|
||||
t.Fatalf("unexpected error: %v", entry.Error)
|
||||
}
|
||||
fromStream.Insert(entry.Handle)
|
||||
}
|
||||
rtest.Equals(t, want, fromStream)
|
||||
}
|
||||
|
||||
func TestAllIndexBlobsEarlyStop(t *testing.T) {
|
||||
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
|
||||
|
||||
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
|
||||
for range 5 {
|
||||
_, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, []byte("test"), restic.ID{}, false)
|
||||
rtest.OK(t, err)
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
|
||||
var count int
|
||||
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
|
||||
rtest.Assert(t, entry.Error == nil, "unexpected error after early stop: %v", entry.Error)
|
||||
count++
|
||||
break
|
||||
}
|
||||
rtest.Equals(t, 1, count)
|
||||
}
|
||||
@@ -586,6 +586,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
|
||||
// unreferenced packs can be safely deleted first
|
||||
if len(plan.removePacksFirst) != 0 {
|
||||
printer.P("deleting unreferenced packs\n")
|
||||
// ignoring errors is fine here as keeping too many packs cannot damage the repository
|
||||
_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacksFirst, restic.PackFile, printer)
|
||||
// forget unused data
|
||||
plan.removePacksFirst = nil
|
||||
@@ -643,6 +644,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
|
||||
|
||||
if len(plan.removePacks) != 0 {
|
||||
printer.P("removing %d old packs", len(plan.removePacks))
|
||||
// ignoring errors is fine here as keeping too many packs cannot damage the repository
|
||||
_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacks, restic.PackFile, printer)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
|
||||
@@ -86,7 +86,7 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
|
||||
blobs := restic.NewBlobSet()
|
||||
|
||||
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
|
||||
entries, err := repo.ListPack(context.TODO(), id, size)
|
||||
entries, _, err := repo.ListPack(context.TODO(), id, size)
|
||||
if err != nil {
|
||||
t.Fatalf("error listing pack %v: %v", id, err)
|
||||
}
|
||||
|
||||
@@ -76,8 +76,9 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
|
||||
|
||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||
if ids.Has(id) {
|
||||
blobs, err := repo.ListPack(ctx, id, size)
|
||||
blobs, _, err := repo.ListPack(ctx, id, size)
|
||||
if err != nil {
|
||||
// ignore errors for broken pack files to be able to salvage as much as possible
|
||||
return nil
|
||||
}
|
||||
packToBlobs[id] = blobs
|
||||
|
||||
@@ -783,7 +783,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
|
||||
// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
|
||||
worker := func() error {
|
||||
for fi := range ch {
|
||||
entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
|
||||
entries, _, err := r.ListPack(wgCtx, fi.ID, fi.Size)
|
||||
if err != nil {
|
||||
debug.Log("unable to list pack file %v", fi.ID.Str())
|
||||
m.Lock()
|
||||
@@ -844,13 +844,8 @@ func (r *Repository) prepareCache() error {
|
||||
|
||||
packs := r.idx.Packs(restic.NewIDSet())
|
||||
|
||||
ids := make(map[string]struct{})
|
||||
for id := range packs {
|
||||
ids[id.String()] = struct{}{}
|
||||
}
|
||||
|
||||
// clear old packs
|
||||
return r.cache.Clear(backend.PackFile, ids)
|
||||
return r.cache.Clear(restic.PackFile, packs)
|
||||
}
|
||||
|
||||
// SearchKey finds a key with the supplied password, afterwards the config is
|
||||
@@ -964,11 +959,12 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
|
||||
})
|
||||
}
|
||||
|
||||
// ListPack returns the list of blobs saved in the pack id.
|
||||
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
|
||||
// ListPack returns the list of blobs saved in the pack id and the length of
|
||||
// the pack header.
|
||||
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, uint32, error) {
|
||||
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
|
||||
|
||||
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
||||
entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
||||
if err != nil {
|
||||
if r.cache != nil {
|
||||
// ignore error as there is not much we can do here
|
||||
@@ -976,9 +972,9 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
|
||||
}
|
||||
|
||||
// retry on error
|
||||
entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
||||
entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
|
||||
}
|
||||
return entries, err
|
||||
return entries, hdrSize, err
|
||||
}
|
||||
|
||||
// Delete calls backend.Delete() if implemented, and returns an error
|
||||
|
||||
@@ -319,8 +319,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
|
||||
var repoFixture = filepath.Join("testdata", "test-repo.tar.gz")
|
||||
|
||||
func TestRepositoryLoadIndex(t *testing.T) {
|
||||
repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
|
||||
defer cleanup()
|
||||
repo, _ := repository.TestFromFixture(t, repoFixture)
|
||||
|
||||
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
|
||||
}
|
||||
@@ -373,8 +372,7 @@ func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length
|
||||
}
|
||||
|
||||
func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
|
||||
repodir, cleanup := rtest.Env(t, repoFixture)
|
||||
defer cleanup()
|
||||
repodir := rtest.Env(t, repoFixture)
|
||||
|
||||
be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2}, t.Logf)
|
||||
rtest.OK(t, err)
|
||||
@@ -479,7 +477,7 @@ func TestListPack(t *testing.T) {
|
||||
return nil
|
||||
}))
|
||||
|
||||
blobs, err := repo.ListPack(context.TODO(), packID, size)
|
||||
blobs, _, err := repo.ListPack(context.TODO(), packID, size)
|
||||
rtest.OK(t, err)
|
||||
rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
|
||||
|
||||
|
||||
@@ -105,11 +105,11 @@ func TestRepositoryWithVersion(t testing.TB, version uint) (*Repository, restic.
|
||||
return repo, &internalRepository{repo}, be
|
||||
}
|
||||
|
||||
func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend, func()) {
|
||||
repodir, cleanup := test.Env(t, repoFixture)
|
||||
func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend) {
|
||||
repodir := test.Env(t, repoFixture)
|
||||
repo, be := TestOpenLocal(t, repodir)
|
||||
|
||||
return repo, be, cleanup
|
||||
return repo, be
|
||||
}
|
||||
|
||||
// TestOpenLocal opens a local repository.
|
||||
|
||||
@@ -24,13 +24,13 @@ func (e *NoIDByPrefixError) Error() string {
|
||||
// Find loads the list of all files of type t and searches for names which
|
||||
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
|
||||
// If more than one is found, nil and ErrMultipleIDMatches is returned.
|
||||
func Find(ctx context.Context, repo Lister, t FileType, prefix string) (ID, error) {
|
||||
func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
|
||||
match := ID{}
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
err := repo.List(ctx, t, func(id ID, _ int64) error {
|
||||
err := be.List(ctx, t, func(id ID, _ int64) error {
|
||||
name := id.String()
|
||||
if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
|
||||
if match.IsNull() {
|
||||
|
||||
@@ -57,6 +57,10 @@ func (h BlobHandle) String() string {
|
||||
return fmt.Sprintf("<%s/%s>", h.Type, h.ID.Str())
|
||||
}
|
||||
|
||||
func NewRandomBlobHandle() BlobHandle {
|
||||
return BlobHandle{ID: NewRandomID(), Type: DataBlob}
|
||||
}
|
||||
|
||||
// BlobType specifies what a blob stored in a pack is.
|
||||
type BlobType uint8
|
||||
|
||||
|
||||
@@ -26,6 +26,11 @@ const MaxRepoVersion = 2
|
||||
// is newly created with Init().
|
||||
const StableRepoVersion = 2
|
||||
|
||||
// JSONUnpackedLoader loads unpacked JSON.
|
||||
type JSONUnpackedLoader interface {
|
||||
LoadJSONUnpacked(context.Context, FileType, ID, interface{}) error
|
||||
}
|
||||
|
||||
// CreateConfig creates a config file with a randomly selected polynomial and
|
||||
// ID.
|
||||
func CreateConfig(version uint) (Config, error) {
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Hash returns the ID for data.
|
||||
@@ -38,6 +40,17 @@ func (id ID) String() string {
|
||||
return hex.EncodeToString(id[:])
|
||||
}
|
||||
|
||||
// NewRandomID returns a randomly generated ID. When reading from rand fails,
|
||||
// the function panics.
|
||||
func NewRandomID() ID {
|
||||
id := ID{}
|
||||
_, err := io.ReadFull(rand.Reader, id[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
const shortStr = 4
|
||||
|
||||
// Str returns the shortened string version of id.
|
||||
|
||||
@@ -32,8 +32,9 @@ type Repository interface {
|
||||
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
|
||||
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
|
||||
ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
|
||||
// ListPack returns the list of blobs saved in the pack id.
|
||||
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
|
||||
// ListPack returns the list of blobs saved in the pack id and the length of
|
||||
// the pack header.
|
||||
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, hdrSize uint32, err error)
|
||||
|
||||
LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
|
||||
LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
// TestParseID parses s as a ID and panics if that fails.
|
||||
@@ -20,18 +18,3 @@ func TestParseID(s string) ID {
|
||||
func TestParseHandle(s string, t BlobType) BlobHandle {
|
||||
return BlobHandle{ID: TestParseID(s), Type: t}
|
||||
}
|
||||
|
||||
func NewRandomBlobHandle() BlobHandle {
|
||||
return BlobHandle{ID: NewRandomID(), Type: DataBlob}
|
||||
}
|
||||
|
||||
// NewRandomID returns a randomly generated ID. When reading from rand fails,
|
||||
// the function panics.
|
||||
func NewRandomID() ID {
|
||||
id := ID{}
|
||||
_, err := io.ReadFull(rand.Reader, id[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
@@ -80,9 +80,12 @@ func extractToFile(buf []byte, filename, target string, printf func(string, ...i
|
||||
return err
|
||||
}
|
||||
if err = newFile.Sync(); err != nil {
|
||||
_ = newFile.Close()
|
||||
_ = os.Remove(newFile.Name())
|
||||
return err
|
||||
}
|
||||
if err = newFile.Close(); err != nil {
|
||||
_ = os.Remove(newFile.Name())
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -87,6 +87,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
|
||||
var msg githubError
|
||||
jerr := json.NewDecoder(res.Body).Decode(&msg)
|
||||
if jerr == nil {
|
||||
_ = res.Body.Close()
|
||||
return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n %v", res.StatusCode, res.Status, msg.Message)
|
||||
}
|
||||
}
|
||||
@@ -137,6 +138,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
_ = res.Body.Close()
|
||||
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
|
||||
+12
-11
@@ -137,10 +137,18 @@ func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
|
||||
|
||||
// Env creates a test environment and extracts the repository fixture.
|
||||
// Returned is the repo path and a cleanup function.
|
||||
func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
|
||||
func Env(t testing.TB, repoFixture string) string {
|
||||
t.Helper()
|
||||
tempdir, err := os.MkdirTemp(TestTempDir, "restic-test-env-")
|
||||
OK(t, err)
|
||||
|
||||
var tempdir string
|
||||
if TestCleanupTempDirs {
|
||||
tempdir = t.TempDir()
|
||||
} else {
|
||||
var err error
|
||||
tempdir, err = os.MkdirTemp(TestTempDir, "restic-test-env-")
|
||||
OK(t, err)
|
||||
t.Logf("leaving temporary directory %v used for test", tempdir)
|
||||
}
|
||||
|
||||
fd, err := os.Open(repoFixture)
|
||||
if err != nil {
|
||||
@@ -150,14 +158,7 @@ func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
|
||||
|
||||
SetupTarTestFixture(t, tempdir, repoFixture)
|
||||
|
||||
return filepath.Join(tempdir, "repo"), func() {
|
||||
if !TestCleanupTempDirs {
|
||||
t.Logf("leaving temporary directory %v used for test", tempdir)
|
||||
return
|
||||
}
|
||||
|
||||
RemoveAll(t, tempdir)
|
||||
}
|
||||
return filepath.Join(tempdir, "repo")
|
||||
}
|
||||
|
||||
func isFile(fi os.FileInfo) bool {
|
||||
|
||||
Reference in New Issue
Block a user