Compare commits

..
39 Commits
Author SHA1 Message Date
Alexander Neumann 6335213501 Set development version for 0.19.1 2026-07-05 09:52:54 +02:00
Alexander Neumann 6aa3a516ce Add version for 0.19.1 2026-07-05 09:52:50 +02:00
Alexander Neumann b1da29738c Generate CHANGELOG.md for 0.19.1 2026-07-05 09:52:30 +02:00
Alexander Neumann 09a1eb6714 Prepare changelog for 0.19.1 2026-07-05 09:52:30 +02:00
Michael Eischer c2b4d6a959 Patch release cherry-pick (#21914) 2026-06-24 20:06:20 +02:00
Michael Eischer 437945bf90 Merge pull request #21911 from restic/patch-release-polish-changelogs
Polish changelogs
2026-06-24 19:48:35 +02:00
Michael Eischerandrawtaz fc422d4145 Apply suggestions from code review
Co-authored-by: rawtaz <rawtaz@users.noreply.github.com>
2026-06-24 19:30:11 +02:00
Michael Eischer f058670219 polish changelogs 2026-06-23 19:55:12 +02:00
Michael Eischer df8454770b Merge pull request #21910 from restic/patch-release-picks
Cherry pick changes for patch release
2026-06-23 19:54:25 +02:00
Michael Eischer c9185dd19c repository/index: ensure progress bar cancelation on early index load failure (#21881) 2026-06-23 19:40:50 +02:00
Ricardo Sawir 32be2e559b fuse: reset treeCache on snapshot reload to fix stale latest symlink (#21873)
The treeCache in SnapshotsDir was never cleared when snapshots were
reloaded. This caused the "latest" symlink to keep pointing to the
previous snapshot even after new snapshots were added.

Add a generation counter to SnapshotsDirStructure that is incremented
whenever the directory structure is rebuilt (in makeDirs). The
treeCache checks this generation on each lookup and resets itself
when the generation changes, ensuring cached nodes (including symlinks)
are refreshed after a snapshot reload.
2026-06-23 19:26:29 +02:00
Eyüp Can Akman add4fa1efb backup: skip source paths that cannot be accessed (#21852) 2026-06-23 19:26:13 +02:00
Michael Eischer c7c3414641 archiver: ignore duplicate but excluded directory entry (#21900) 2026-06-23 19:26:10 +02:00
Michael Eischer 8ef295e2f5 snapshots: revert default --lastest behavior to pre-0.19.0
the changed behavior now only applies when using `--group-by`.
2026-06-23 19:15:14 +02:00
Michael Eischer bc714d1864 snapshots: cleanup handling of deprecated --last option 2026-06-23 19:15:14 +02:00
Michael Eischer debf95f58d backend/sftp: fix removing files on windows sftp 2026-06-23 19:14:54 +02:00
Michael Eischer 27c44c11aa backend/sftp: fix error reporting of Save method 2026-06-23 19:14:54 +02:00
Michael Eischer 93f695a803 backend/rest: fix flaky test cleanup (#21901) 2026-06-23 19:14:54 +02:00
Michael Eischer 3e21d1fea6 fs: print missing error if xattr cannot be read (#21896) 2026-06-23 19:13:39 +02:00
Michael Eischer 8ee674da55 repository: show correct error if zstd decoder cannot be initialized 2026-06-23 19:13:39 +02:00
Michael Eischer c21f0fabc6 selfupdate: improve error handling 2026-06-23 19:13:39 +02:00
Michael Eischer 6aaa2eace4 prune/repair pack: document ignored errors 2026-06-23 19:13:39 +02:00
Michael Eischer f2d614db56 backend/util: fix errorhandling in DefaultDelete test helper 2026-06-23 19:13:39 +02:00
Michael Eischer ed655b476a backend/sftp: fix file handle leak if chmod for a file fails 2026-06-23 19:13:39 +02:00
Michael Eischer 80f694b32e backend/rest: correctly close body on out of bounds read 2026-06-23 19:13:39 +02:00
Michael Eischer eaa835becd backend/b2: correctly close file if too short 2026-06-23 19:13:39 +02:00
Michael Eischer 7aa82d68b4 forget: fix error handling if snapshot key cannot be marshaled 2026-06-23 19:13:39 +02:00
Michael Eischer a3fa3eb182 ensure reliable cleanup of test repository (#21880) 2026-06-23 19:12:08 +02:00
Michael Eischer cb24c4f566 snapshots: show location name instead of current timezone (#21877) 2026-06-23 19:11:19 +02:00
Michael Eischer d6fd4ae26f mount: fix crash in mountpoint validation (#21879) 2026-06-23 19:09:57 +02:00
Michael Eischer 4134bdfac2 stats: hide progress bar for json output (#21871) 2026-06-23 19:08:36 +02:00
Michael Eischer c08e7fd956 mount: unify mountpoint validation error messages 2026-06-23 19:08:36 +02:00
Michael Eischer d07bcd50da mount: factor out mountpoint validation into helper 2026-06-23 19:08:36 +02:00
Srigovind Nayak e6b9118f51 doc: note that mountpoint must not overlap the repository
Document the new restriction added so users encountering the error
message have a reference, and so the constraint is visible before they
hit it.
2026-06-23 19:08:12 +02:00
Srigovind Nayak 82a665384b chore: update changelog for issue 5234 2026-06-23 19:08:12 +02:00
Srigovind Nayak 1e249b0da1 mount: refuse mountpoints that overlap the local repository
Mounting a local repository onto its own directory caused the FUSE
  server to read its own backend files through the mount it had just
  created, deadlocking the kernel. `umount` then reported "Device or
  resource busy" and recovery required a reboot that took several
  minutes. The same shape occurs when the mountpoint is nested inside
  the repository directory, or when the repository directory is nested
  inside the mountpoint.

  The mount command now resolves both paths via filepath.Abs and
  filepath.EvalSymlinks and refuses with a fatal error if either path
  equals or contains the other. The check runs before the repository
  lock is acquired so an overlap fails fast. Only the local backend is
  checked; remote backends cannot shadow the mountpoint directory.
2026-06-23 19:08:12 +02:00
Michael Eischer 5967f9a894 upgrade compress library 2026-06-23 19:07:56 +02:00
Michael Eischer 1733dd56b9 repository: fix zero-sized blobs in v2 repos 2026-06-23 19:07:56 +02:00
Michael Eischer ed39b60e7d fix version set by helper after release (#21865) 2026-06-23 19:06:55 +02:00
77 changed files with 1755 additions and 1102 deletions
+1 -7
View File
@@ -32,6 +32,7 @@ linters:
backend-imports:
files:
- "**/internal/backend/**"
- "!**/internal/backend/cache/**"
- "!**/internal/backend/test/**"
- "!**/*_test.go"
deny:
@@ -39,13 +40,6 @@ linters:
desc: "internal/restic should not be imported to keep the architectural layers intact"
- pkg: "github.com/restic/restic/internal/repository"
desc: "internal/repository should not be imported to keep the architectural layers intact"
repository-internals:
files:
- "**"
- "!**/internal/repository/**"
deny:
- pkg: "github.com/restic/restic/internal/repository/"
desc: "packages below internal/repository should not be imported to not depend on repository internals"
importas:
alias:
- pkg: github.com/restic/restic/internal/test
+111
View File
@@ -1,5 +1,6 @@
# Table of Contents
* [Changelog for 0.19.1](#changelog-for-restic-0191-2026-07-05)
* [Changelog for 0.19.0](#changelog-for-restic-0190-2026-06-09)
* [Changelog for 0.18.1](#changelog-for-restic-0181-2025-09-21)
* [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
@@ -41,6 +42,116 @@
* [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
# Changelog for restic 0.19.1 (2026-07-05)
The following sections list the changes in restic 0.19.1 relevant to
restic users. The changes are ordered by importance.
## Summary
* Fix #5234: Prevent mounting over the repository directory
* Fix #5667: Skip inaccessible `backup` source paths
* Fix #5722: Update `mount` latest symlink after snapshot reload
* Fix #21866: Hide `stats` progress bar in JSON mode
* Fix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
* Fix #21876: Show timezone location in `snapshots` output
* Fix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
* Fix #21895: Remove read-only files via the SFTP backend on Windows servers
* Fix #21899: Make `backup` respect excludes for duplicate directory entries
## Details
* Bugfix #5234: Prevent mounting over the repository directory
Using a local repository directory as the `mount` target — or a path that
contains it, or that it contains — caused the FUSE server to read its own
backend files through the new mount, deadlocking the kernel and requiring a long
reboot to recover.
Restic now resolves both paths and refuses any such overlap with a clear error
before mounting.
https://github.com/restic/restic/issues/5234
https://github.com/restic/restic/pull/5348
* Bugfix #5667: Skip inaccessible `backup` source paths
The `backup` command only skipped source paths that did not exist. A path that
could not be accessed for another reason, such as a malformed path on Windows,
was kept and produced an empty snapshot. Restic now skips any such path and
aborts if none remain.
https://github.com/restic/restic/issues/5667
https://github.com/restic/restic/pull/21852
* Bugfix #5722: Update `mount` latest symlink after snapshot reload
When `restic mount` was kept running while new snapshots were created, the new
snapshots appeared in the mountpoint, but the `latest` symlink could still point
to the previously latest snapshot. Restic now invalidates the cached snapshot
directory entries after a snapshot reload so that `latest` points to the newest
snapshot.
https://github.com/restic/restic/issues/5722
https://github.com/restic/restic/pull/21873
* Bugfix #21866: Hide `stats` progress bar in JSON mode
Since restic 0.19.0, the `stats` command shows a progress bar. This progress bar
was unintentionally displayed also when using the `--json` option, mixing
regular text output with JSON. This is now fixed.
https://github.com/restic/restic/issues/21866
https://github.com/restic/restic/pull/21871
* Bugfix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>` to
no longer group snapshots by host and paths by default.
The `snapshots --latest <n>` command now again uses the old behavior of grouping
by host and paths when `--group-by` is not specified. However, when specifying
`--group-by` the output is still grouped as requested, as in restic 0.19.0.
https://github.com/restic/restic/issues/21869
https://github.com/restic/restic/pull/21875
* Bugfix #21876: Show timezone location in `snapshots` output
With restic 0.19.0, the `snapshots` command printed the current timezone when
listing snapshots. However, that timezone label might change during the year,
for example with daylight saving time. Restic now prints a more consistent and
shorter version of the text.
https://github.com/restic/restic/pull/21876
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
* Bugfix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
the repository. If restic was unable to stat the mountpoint, this would result
in a crash. This has now been fixed to correctly return an error instead.
https://github.com/restic/restic/pull/21879
* Bugfix #21895: Remove read-only files via the SFTP backend on Windows servers
Since restic 0.19.0, repository files on the SFTP backend are marked read-only
after save. On Windows SFTP servers, removing them failed with a permission
error. The SFTP backend now clears the read-only flag before removing the file.
https://github.com/restic/restic/issues/21895
https://github.com/restic/restic/pull/21897
* Bugfix #21899: Make `backup` respect excludes for duplicate directory entries
Since restic 0.19.0, backing up a directory with duplicate directory entries
always produced "Warning: at least one source file could not be read", even when
those files were excluded. This has now been fixed.
https://github.com/restic/restic/issues/21899
https://github.com/restic/restic/pull/21900
# Changelog for restic 0.19.0 (2026-06-09)
The following sections list the changes in restic 0.19.0 relevant to
restic users. The changes are ordered by importance.
+1 -1
View File
@@ -1 +1 @@
0.19.0-dev
0.19.1-dev
+8
View File
@@ -0,0 +1,8 @@
Bugfix: Hide `stats` progress bar in JSON mode
Since restic 0.19.0, the `stats` command shows a progress bar. This
progress bar was unintentionally displayed also when using the `--json`
option, mixing regular text output with JSON. This is now fixed.
https://github.com/restic/restic/issues/21866
https://github.com/restic/restic/pull/21871
+12
View File
@@ -0,0 +1,12 @@
Bugfix: Restore old behavior of `snapshots --latest <n>` without `--group-by`
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>`
to no longer group snapshots by host and paths by default.
The `snapshots --latest <n>` command now again uses the old behavior of
grouping by host and paths when `--group-by` is not specified. However, when
specifying `--group-by` the output is still grouped as requested, as in restic
0.19.0.
https://github.com/restic/restic/issues/21869
https://github.com/restic/restic/pull/21875
+9
View File
@@ -0,0 +1,9 @@
Bugfix: Remove read-only files via the SFTP backend on Windows servers
Since restic 0.19.0, repository files on the SFTP backend are marked
read-only after save. On Windows SFTP servers, removing them failed
with a permission error. The SFTP backend now clears the read-only flag
before removing the file.
https://github.com/restic/restic/issues/21895
https://github.com/restic/restic/pull/21897
+8
View File
@@ -0,0 +1,8 @@
Bugfix: Make `backup` respect excludes for duplicate directory entries
Since restic 0.19.0, backing up a directory with duplicate directory
entries always produced "Warning: at least one source file could not be
read", even when those files were excluded. This has now been fixed.
https://github.com/restic/restic/issues/21899
https://github.com/restic/restic/pull/21900
+12
View File
@@ -0,0 +1,12 @@
Bugfix: Prevent mounting over the repository directory
Using a local repository directory as the `mount` target — or a path
that contains it, or that it contains — caused the FUSE server to
read its own backend files through the new mount, deadlocking the
kernel and requiring a long reboot to recover.
Restic now resolves both paths and refuses any such overlap with a
clear error before mounting.
https://github.com/restic/restic/issues/5234
https://github.com/restic/restic/pull/5348
+9
View File
@@ -0,0 +1,9 @@
Bugfix: Skip inaccessible `backup` source paths
The `backup` command only skipped source paths that did not exist. A path that
could not be accessed for another reason, such as a malformed path on Windows,
was kept and produced an empty snapshot. Restic now skips any such path and
aborts if none remain.
https://github.com/restic/restic/issues/5667
https://github.com/restic/restic/pull/21852
+10
View File
@@ -0,0 +1,10 @@
Bugfix: Update `mount` latest symlink after snapshot reload
When `restic mount` was kept running while new snapshots were
created, the new snapshots appeared in the mountpoint, but the `latest`
symlink could still point to the previously latest snapshot. Restic now
invalidates the cached snapshot directory entries after a snapshot reload so
that `latest` points to the newest snapshot.
https://github.com/restic/restic/issues/5722
https://github.com/restic/restic/pull/21873
+9
View File
@@ -0,0 +1,9 @@
Bugfix: Show timezone location in `snapshots` output
With restic 0.19.0, the `snapshots` command printed the current timezone when
listing snapshots. However, that timezone label might change during the year,
for example with daylight saving time. Restic now prints a more consistent and
shorter version of the text.
https://github.com/restic/restic/pull/21876
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
+7
View File
@@ -0,0 +1,7 @@
Bugfix: Prevent crash in mountpoint validation if mountpoint is inaccessible
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
the repository. If restic was unable to stat the mountpoint, this would result
in a crash. This has now been fixed to correctly return an error instead.
https://github.com/restic/restic/pull/21879
+8 -4
View File
@@ -171,13 +171,17 @@ var ErrInvalidSourceData = errors.New("at least one source file could not be rea
// ErrNoSourceData is used to report that no source data was found
var ErrNoSourceData = errors.Fatal("all source directories/files do not exist")
// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
// filterExisting returns the items that exist and can be accessed. It returns
// ErrNoSourceData if none remain, or ErrInvalidSourceData if some were skipped.
func filterExisting(items []string, warnf func(msg string, args ...interface{})) (result []string, err error) {
for _, item := range items {
_, err := fs.Lstat(item)
if errors.Is(err, os.ErrNotExist) {
warnf("%v does not exist, skipping\n", item)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
warnf("%v does not exist, skipping\n", item)
} else {
warnf("%v cannot be accessed, skipping\n", item)
}
continue
}
+23
View File
@@ -10,6 +10,7 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/errors"
rtest "github.com/restic/restic/internal/test"
)
@@ -76,6 +77,28 @@ func TestCollectTargets(t *testing.T) {
rtest.Assert(t, err == ErrInvalidSourceData, "expected error when not all targets exist")
}
func TestFilterExistingUnreadable(t *testing.T) {
dir := rtest.TempDir(t)
existing := filepath.Join(dir, "existing")
rtest.OK(t, os.Mkdir(existing, 0755))
file := filepath.Join(dir, "file")
rtest.OK(t, os.WriteFile(file, []byte("x"), 0600))
// Regression test for #5667. A target whose Lstat fails with an error other
// than ErrNotExist must be skipped (ENOTDIR on unix, NUL byte everywhere).
for _, unreadable := range []string{filepath.Join(file, "child"), "invalid\x00path"} {
result, err := filterExisting([]string{unreadable}, t.Logf)
rtest.Assert(t, errors.Is(err, ErrNoSourceData), "input %q: expected ErrNoSourceData; got %v", unreadable, err)
rtest.Assert(t, len(result) == 0, "input %q: expected no targets; got %v", unreadable, result)
result, err = filterExisting([]string{existing, unreadable}, t.Logf)
rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "input %q: expected ErrInvalidSourceData; got %v", unreadable, err)
rtest.Equals(t, []string{existing}, result)
}
}
func TestReadFilenamesRaw(t *testing.T) {
// These should all be returned exactly as-is.
expected := []string{
+1 -1
View File
@@ -307,7 +307,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
go chkr.Packs(ctx, errChan)
for err := range errChan {
var packErr *repository.ErrPackMetadata
var packErr *repository.PackError
if errors.As(err, &packErr) {
if packErr.Orphaned {
orphanedPacks++
+1 -1
View File
@@ -103,7 +103,7 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
defer unlock()
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
blobs, err := repo.ListPack(context.TODO(), id, size)
blobs, _, err := repo.ListPack(context.TODO(), id, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
+365 -10
View File
@@ -4,19 +4,31 @@ package main
import (
"context"
"crypto/aes"
"crypto/cipher"
"encoding/json"
"fmt"
"io"
"os"
"runtime"
"sync"
"time"
"github.com/klauspost/compress/zstd"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
)
func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) {
@@ -116,6 +128,61 @@ func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io
})
}
// Pack is the struct used in printPacks.
type Pack struct {
Name string `json:"name"`
Blobs []Blob `json:"blobs"`
}
// Blob is the struct used in printPacks.
type Blob struct {
Type restic.BlobType `json:"type"`
Length uint `json:"length"`
ID restic.ID `json:"id"`
Offset uint `json:"offset"`
}
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
printer.E("error for pack %v: %v", id.Str(), err)
return nil
}
p := Pack{
Name: id.String(),
Blobs: make([]Blob, len(blobs)),
}
for i, blob := range blobs {
p.Blobs[i] = Blob{
Type: blob.Type,
Length: blob.Length,
ID: blob.ID,
Offset: blob.Offset,
}
}
m.Lock()
defer m.Unlock()
return prettyPrintJSON(wr, p)
})
}
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id)
if err != nil {
return err
}
return idx.Dump(wr)
})
}
func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
@@ -133,11 +200,11 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
switch tpe {
case "indexes":
return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
case "snapshots":
return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
case "packs":
return repository.DumpPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
case "all":
printer.S("snapshots:")
err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
@@ -146,7 +213,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
}
printer.S("indexes:")
err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
if err != nil {
return err
}
@@ -157,6 +224,225 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
}
}
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
if bytewise {
printer.S(" trying to repair blob by finding a broken byte")
} else {
printer.S(" trying to repair blob with single bit flip")
}
ch := make(chan int)
var wg errgroup.Group
done := make(chan struct{})
var fixed []byte
var found bool
workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ {
wg.Go(func() error {
// make a local copy of the buffer
buf := make([]byte, len(input))
copy(buf, input)
testFlip := func(idx int, pattern byte) bool {
// flip bits
buf[idx] ^= pattern
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil {
printer.S("")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext))
close(done)
found = true
fixed = plaintext
return true
}
// flip bits back
buf[idx] ^= pattern
return false
}
for i := range ch {
if bytewise {
for j := 0; j < 255; j++ {
if testFlip(i, byte(j)) {
return nil
}
}
} else {
for j := 0; j < 7; j++ {
// flip each bit once
if testFlip(i, (1 << uint(j))) {
return nil
}
}
}
}
return nil
})
}
wg.Go(func() error {
defer close(ch)
start := time.Now()
info := time.Now()
for i := range input {
select {
case ch <- i:
case <-done:
printer.S(" done after %v", time.Since(start))
return nil
}
if time.Since(info) > time.Second {
secs := time.Since(start).Seconds()
gps := float64(i) / secs
remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now()
}
}
return nil
})
err := wg.Wait()
if err != nil {
panic("all go routines can only return nil")
}
if !found {
printer.S("\n blob could not be repaired")
}
return fixed
}
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
// strip signature at the end
l := len(buf)
nonce, ct := buf[:16], buf[16:l-16]
out := make([]byte, len(ct))
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCTR(c, nonce)
e.XORKeyStream(out, ct)
return out
}
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
// allow processing broken pack files
if pack == nil {
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(pack) {
printer.E("skipping truncated blob")
continue
}
buf := pack[blob.Offset : blob.Offset+blob.Length]
key := repo.Key()
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
outputPrefix := ""
filePrefix := ""
if err != nil {
printer.E("error decrypting blob: %v", err)
if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
if err != nil {
return err
}
continue
}
}
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
printer.S(" failed to decompress blob %v", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
}
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
if err != nil {
return err
}
}
if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
printer.S(" uploaded %v %v", blob.Type, id)
}
}
return nil
})
return err
}
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename)
if err != nil {
return err
}
_, err = f.Write(plain)
if err != nil {
_ = f.Close()
return err
}
err = f.Close()
if err != nil {
return err
}
printer.S("decrypt of blob %v stored at %v", id, filename)
return nil
}
func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
@@ -192,14 +478,8 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
return err
}
examineOpts := repository.ExaminePackOptions{
TryRepair: opts.TryRepair,
RepairByte: opts.RepairByte,
ExtractPack: opts.ExtractPack,
ReuploadBlobs: opts.ReuploadBlobs,
}
for _, id := range ids {
err := repository.ExaminePack(ctx, repo, id, examineOpts, printer)
err := examinePack(ctx, opts, repo, id, printer)
if err != nil {
printer.E("error: %v", err)
}
@@ -209,3 +489,78 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
}
return nil
}
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
printer.S("examine %v", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
printer.S(" file size is %v", len(buf))
gotID := restic.Hash(buf)
if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID)
} else {
printer.S(" hash for file content matches")
}
printer.S(" ========================================")
printer.S(" looking for info in the indexes")
blobsLoaded := false
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
continue
}
checkPackSize(blobs, len(buf), printer)
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
if err != nil {
printer.E("error: %v", err)
} else {
blobsLoaded = true
}
}
printer.S(" ========================================")
printer.S(" inspect the pack itself")
blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
checkPackSize(blobs, len(buf), printer)
if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer)
}
return nil
}
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
// track current size and offset
var size, offset uint64
blobs.Sort()
for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
}
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
}
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
} else {
printer.S(" file sizes match")
}
}
+1 -1
View File
@@ -472,7 +472,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
delete(packIDs, idStr)
}
debug.Log("Found pack %s", idStr)
blobs, err := f.repo.ListPack(ctx, id, size)
blobs, _, err := f.repo.ListPack(ctx, id, size)
if err != nil {
return err
}
+1 -1
View File
@@ -261,7 +261,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
}
var key data.SnapshotGroupKey
if json.Unmarshal([]byte(k), &key) != nil {
if err := json.Unmarshal([]byte(k), &key); err != nil {
return err
}
+12 -7
View File
@@ -6,7 +6,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
@@ -69,13 +69,18 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T
case "locks":
t = restic.LockFile
case "blobs":
for entry := range repository.AllIndexBlobs(ctx, repo, repo) {
if entry.Error != nil {
return entry.Error
return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
if err != nil {
return err
}
printer.S("%v %v", entry.Handle.Type, entry.Handle.ID)
}
return nil
for blobs := range idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
}
printer.S("%v %v", blobs.Type, blobs.ID)
}
return nil
})
default:
return errors.Fatal("invalid type")
}
+93 -16
View File
@@ -6,6 +6,7 @@ import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
@@ -13,6 +14,8 @@ import (
"github.com/spf13/pflag"
"golang.org/x/sys/unix"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -131,22 +134,8 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
}
mountpoint := args[0]
// Check the existence of the mount point at the earliest stage to
// prevent unnecessary computations while opening the repository.
stat, err := os.Stat(mountpoint)
if errors.Is(err, os.ErrNotExist) {
printer.P("Mountpoint %s doesn't exist", mountpoint)
return errors.Fatal("invalid mountpoint")
} else if !stat.IsDir() {
printer.P("Mountpoint %s is not a directory", mountpoint)
return errors.Fatal("invalid mountpoint")
}
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
if err != nil {
printer.P("Mountpoint %s is not writeable or not executable", mountpoint)
return errors.Fatal("inaccessible mountpoint")
if err := validateMountpoint(mountpoint, gopts); err != nil {
return err
}
debug.Log("start mount")
@@ -230,3 +219,91 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
return err
}
func validateMountpoint(mountpoint string, gopts global.Options) error {
// Check the existence of the mount point at the earliest stage to
// prevent unnecessary computations while opening the repository.
stat, err := os.Stat(mountpoint)
if errors.Is(err, os.ErrNotExist) {
return errors.Fatal(fmt.Sprintf("mountpoint %s does not exist", mountpoint))
} else if err != nil {
return errors.Fatal(fmt.Sprintf("mountpoint %s is inaccessible: %v", mountpoint, err))
} else if !stat.IsDir() {
return errors.Fatal(fmt.Sprintf("mountpoint %s is not a directory", mountpoint))
}
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
if err != nil {
return errors.Fatal(fmt.Sprintf("mountpoint %s is not writeable or not executable", mountpoint))
}
// Refuse to mount onto (or under, or over) the local repository directory.
// Doing so makes the FUSE server read its own backend files through the
// mount it just created, deadlocking the kernel (GH #5234).
loc, err := location.Parse(gopts.Backends, gopts.Repo)
if err != nil {
return err
}
if loc.Scheme == "local" {
if err := checkMountpointOverlap(loc.Config.(*local.Config).Path, mountpoint); err != nil {
return err
}
}
return nil
}
// checkMountpointOverlap returns an error.Fatal if the local repository at
// repoPath and the mountpoint overlap: equal paths, mountpoint nested inside
// the repo, or the repo nested inside the mountpoint. Any overlap deadlocks
// the FUSE server (GH #5234).
func checkMountpointOverlap(repoPath, mountpoint string) error {
rp, err := resolvePath(repoPath)
if err != nil {
return err
}
mp, err := resolvePath(mountpoint)
if err != nil {
return err
}
const tail = "; refusing to mount to avoid deadlocking the FUSE server"
switch {
case rp == mp:
return errors.Fatal(fmt.Sprintf("mountpoint %s is the local repository directory%s", mp, tail))
case isInside(rp, mp):
return errors.Fatal(fmt.Sprintf("mountpoint %s is inside the local repository directory %s%s", mp, rp, tail))
case isInside(mp, rp):
return errors.Fatal(fmt.Sprintf("local repository directory %s is inside the mountpoint %s%s", rp, mp, tail))
}
return nil
}
// resolvePath returns p as an absolute, symlink-resolved path. If EvalSymlinks
// fails (e.g. the path does not fully exist), it falls back to the absolute
// form: overlap detection is best-effort and we'd rather refuse a clear
// overlap than abort on an unrelated stat error.
func resolvePath(p string) (string, error) {
abs, err := filepath.Abs(p)
if err != nil {
return "", err
}
resolved, err := filepath.EvalSymlinks(abs)
if err != nil {
return abs, nil
}
return resolved, nil
}
// isInside reports whether child is strictly nested inside parent. Both paths
// must already be cleaned and absolute. Equal paths return false; the caller
// handles equality separately so it can produce a distinct error message.
func isInside(parent, child string) bool {
rel, err := filepath.Rel(parent, child)
if err != nil {
return false
}
if rel == "." || rel == ".." {
return false
}
return !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}
+57
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -212,6 +213,62 @@ func TestMount(t *testing.T) {
checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 4)
}
func TestCheckMountpointOverlap(t *testing.T) {
tempdir := t.TempDir()
repo := filepath.Join(tempdir, "repo")
repoSub := filepath.Join(repo, "sub")
sibling := filepath.Join(tempdir, "mnt")
for _, d := range []string{repo, repoSub, sibling} {
rtest.OK(t, os.MkdirAll(d, 0700))
}
cases := []struct {
name string
repo string
mount string
wantSub string // substring of expected error; empty means expect nil
}{
{"equal", repo, repo, "is the local repository directory"},
{"mount inside repo", repo, repoSub, "is inside the local repository directory"},
{"repo inside mount", repoSub, repo, "is inside the mountpoint"},
{"disjoint", repo, sibling, ""},
{"prefix-not-subpath", repo, repo + "-other", ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rtest.OK(t, os.MkdirAll(tc.mount, 0700))
err := checkMountpointOverlap(tc.repo, tc.mount)
if tc.wantSub == "" {
rtest.OK(t, err)
return
}
if err == nil {
t.Fatalf("expected error containing %q, got nil", tc.wantSub)
}
if !strings.Contains(err.Error(), tc.wantSub) {
t.Fatalf("error %q does not contain %q", err.Error(), tc.wantSub)
}
})
}
}
func TestCheckMountpointOverlapSymlink(t *testing.T) {
tempdir := t.TempDir()
repo := filepath.Join(tempdir, "repo")
rtest.OK(t, os.MkdirAll(repo, 0700))
link := filepath.Join(tempdir, "link-to-repo")
rtest.OK(t, os.Symlink(repo, link))
err := checkMountpointOverlap(repo, link)
if err == nil {
t.Fatal("expected overlap error when mountpoint is a symlink to repo, got nil")
}
if !strings.Contains(err.Error(), "is the local repository directory") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestMountSameTimestamps(t *testing.T) {
if !rtest.RunFuseTest {
t.Skip("Skipping fuse tests")
@@ -11,6 +11,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -60,6 +61,15 @@ func TestRebuildIndex(t *testing.T) {
testRebuildIndex(t, nil)
}
func TestRebuildIndexAlwaysFull(t *testing.T) {
indexFull := index.Full
defer func() {
index.Full = indexFull
}()
index.Full = func(*index.Index) bool { return true }
testRebuildIndex(t, nil)
}
// indexErrorBackend modifies the first index after reading.
type indexErrorBackend struct {
backend.Backend
+59 -12
View File
@@ -38,6 +38,9 @@ Exit status is 12 if the password is incorrect.
`,
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
PreRunE: func(_ *cobra.Command, _ []string) error {
return opts.Finalize()
},
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runSnapshots(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
@@ -52,7 +55,7 @@ Exit status is 12 if the password is incorrect.
type SnapshotOptions struct {
data.SnapshotFilter
Compact bool
Last bool // This option should be removed in favour of Latest.
last bool // Deprecated in favour of Latest.
Latest int
GroupBy data.SnapshotGroupByOptions
}
@@ -60,7 +63,7 @@ type SnapshotOptions struct {
func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format")
f.BoolVar(&opts.Last, "last", false, "only show the last snapshot for each host and path")
f.BoolVar(&opts.last, "last", false, "only show the last snapshot for each host and path")
err := f.MarkDeprecated("last", "use --latest 1")
if err != nil {
// MarkDeprecated only returns an error when the flag is not found
@@ -70,6 +73,13 @@ func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma")
}
func (opts *SnapshotOptions) Finalize() error {
if opts.last && opts.Latest == 0 {
opts.Latest = 1
}
return nil
}
func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
@@ -95,12 +105,12 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
return ctx.Err()
}
if opts.Last {
// This branch should be removed in the same time
// that --last.
list = filterLatestSnapshotsInGroup(list, 1)
} else if opts.Latest > 0 {
list = filterLatestSnapshotsInGroup(list, opts.Latest)
if opts.Latest > 0 {
if grouped {
list = filterLatestSnapshotsInGroup(list, opts.Latest)
} else {
list = filterLatestSnapshots(list, opts.Latest)
}
}
sort.Sort(sort.Reverse(list))
snapshotGroups[k] = list
@@ -134,6 +144,43 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
return nil
}
// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
Hostname string
JoinedPaths string
}
// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *data.Snapshot) filterLastSnapshotsKey {
// Shallow slice copy
var paths = make([]string, len(sn.Paths))
copy(paths, sn.Paths)
sort.Strings(paths)
return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}
// filterLatestSnapshots filters a list of snapshots to only return
// the limit last entries for each hostname and path. If the snapshot
// contains multiple paths, they will be joined and treated as one
// item.
func filterLatestSnapshots(list data.Snapshots, limit int) data.Snapshots {
// Sort the snapshots so that the newer ones are listed first
sort.SliceStable(list, func(i, j int) bool {
return list[i].Time.After(list[j].Time)
})
var results data.Snapshots
seen := make(map[filterLastSnapshotsKey]int)
for _, sn := range list {
key := newFilterLastSnapshotsKey(sn)
if seen[key] < limit {
seen[key]++
results = append(results, sn)
}
}
return results
}
// filterLatestSnapshotsInGroup filters a list of snapshots to only return
// the `limit` last entries. It is assumed that the snapshot list only contains
// one group of snapshots.
@@ -245,11 +292,11 @@ func PrintSnapshots(stdout io.Writer, list data.Snapshots, reasons []data.KeepRe
// Each snapshot can be registered in different timezones,
// but we display them all in local timezone on this output.
footer := fmt.Sprintf("%d snapshots", len(list))
zoneName, _ := time.Now().Local().Zone()
if zoneName != "" {
footer = fmt.Sprintf("Timestamps shown in %s timezone\n%s", zoneName, footer)
zoneName := time.Now().Local().Location().String()
if zoneName == "Local" {
zoneName = "local time"
}
tab.AddFooter(footer)
tab.AddFooter(fmt.Sprintf("Timestamps shown in %s\n%s", zoneName, footer))
if multiline {
// print an additional blank line between snapshots
+33 -5
View File
@@ -35,10 +35,7 @@ func testRunSnapshots(t testing.TB, gopts global.Options) (newest *Snapshot, sna
return
}
func TestSnapshotsGroupByAndLatest(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
func snapshotsGroupTestData(t *testing.T, env *testEnvironment, keepPath bool) string {
testSetupBackupData(t, env)
// two backups on the same host but with different paths
opts := BackupOptions{Host: "testhost", TimeStamp: time.Now().Format(time.DateTime)}
@@ -46,9 +43,22 @@ func TestSnapshotsGroupByAndLatest(t *testing.T) {
// Use later timestamp for second backup
opts.TimeStamp = time.Now().Add(time.Second).Format(time.DateTime)
snapshotsIDs := loadSnapshotMap(t, env.gopts)
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0"}, opts, env.gopts)
targets := []string{"testdata/0"}
if keepPath {
targets = []string{"testdata"}
}
testRunBackup(t, filepath.Dir(env.testdata), targets, opts, env.gopts)
_, secondSnapshotID := lastSnapshot(snapshotsIDs, loadSnapshotMap(t, env.gopts))
return secondSnapshotID
}
func TestSnapshotsGroupByAndLatest(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
secondSnapshotID := snapshotsGroupTestData(t, env, false)
buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
gopts.JSON = true
// only group by host but not path
@@ -65,3 +75,21 @@ func TestSnapshotsGroupByAndLatest(t *testing.T) {
rtest.Assert(t, len(snapshots[0].Snapshots) == 1, "expected only one latest snapshot, got %d", len(snapshots[0].Snapshots))
rtest.Equals(t, snapshots[0].Snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
}
func TestSnapshotsLatest(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
secondSnapshotID := snapshotsGroupTestData(t, env, true)
buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
gopts.JSON = true
opts := SnapshotOptions{Latest: 1}
return runSnapshots(ctx, opts, gopts, []string{}, gopts.Term)
})
rtest.OK(t, err)
snapshots := []Snapshot{}
rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
rtest.Assert(t, len(snapshots) == 1, "expected only one snapshot, got %d", len(snapshots))
rtest.Equals(t, snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
}
+7 -2
View File
@@ -141,7 +141,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
snapshots = append(snapshots, sn)
}
statsProgress := newStatsProgress(term, uint64(len(snapshots)))
statsProgress := newStatsProgress(term, !gopts.JSON, uint64(len(snapshots)))
updater := progress.NewUpdater(ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()), func(runtime time.Duration, final bool) {
statsProgress.printProgress(runtime, final)
@@ -375,6 +375,7 @@ type statsProgress struct {
term ui.Terminal
m sync.Mutex
snapshotCount uint64
show bool
processedSnapshotCount uint64
processedFileCount uint64
@@ -382,14 +383,18 @@ type statsProgress struct {
processedSize uint64
}
func newStatsProgress(term ui.Terminal, snapshotCount uint64) *statsProgress {
func newStatsProgress(term ui.Terminal, show bool, snapshotCount uint64) *statsProgress {
return &statsProgress{
term: term,
show: show,
snapshotCount: snapshotCount,
}
}
func (s *statsProgress) printProgress(runtime time.Duration, final bool) {
if !s.show {
return
}
s.m.Lock()
progressBase := s.processedSnapshotCount
+10 -1
View File
@@ -66,7 +66,7 @@ func TestSizeHistogramString(t *testing.T) {
func TestStatsProgress(t *testing.T) {
term := &ui.MockTerminal{}
progress := newStatsProgress(term, 2)
progress := newStatsProgress(term, true, 2)
progress.printProgress(0*time.Second, false)
rtest.Equals(t, []string{"[0:00] 0.00% 0 / 2 snapshots, 0 B"}, term.Output)
@@ -88,3 +88,12 @@ func TestStatsProgress(t *testing.T) {
progress.printProgress(20*time.Second, true)
rtest.Equals(t, []string{"[0:20] 100.00% 2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output)
}
func TestStatsProgressJSON(t *testing.T) {
term := &ui.MockTerminal{}
progress := newStatsProgress(term, false, 2)
progress.printProgress(0*time.Second, false)
// JSON output is not available yet, so just make sure to not break normal json output
rtest.Equals(t, nil, term.Output)
}
+6
View File
@@ -208,6 +208,12 @@ command needs to be in the ``PATH``. On macOS, you need `FUSE-T
<https://www.fuse-t.org/>`__ or `FUSE for macOS <https://osxfuse.github.io/>`__.
On FreeBSD, you may need to install FUSE and load the kernel module (``kldload fuse``).
.. note:: The mountpoint must not overlap the local repository directory.
Using the repository directory itself, a subdirectory of it, or a parent
of it as the mountpoint causes the FUSE server to read its own backend
files through the new mount and deadlock the kernel. ``restic mount``
detects this and refuses such mountpoints.
Restic supports storage and preservation of hard links. However, since
hard links exist in the scope of a filesystem by definition, restoring
hard links from a FUSE mount should be done by a program that preserves
+9
View File
@@ -320,6 +320,8 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
finder := data.NewTreeFinder(previous)
defer finder.Close()
var lastExcluded string
for _, name := range names {
// test if context has been cancelled
if ctx.Err() != nil {
@@ -327,6 +329,12 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
return futureNode{}, ctx.Err()
}
if name == lastExcluded {
// Skip duplicate directory entry if it was already excluded.
// This avoids printing errors about duplicate directory entries even though the entry in question is ignored.
continue
}
pathname := arch.FS.Join(dir, name)
oldNode, err := finder.Find(name)
err = arch.error(pathname, err)
@@ -348,6 +356,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
}
if excluded {
lastExcluded = name
continue
}
+84
View File
@@ -894,6 +894,90 @@ func TestArchiverSaveDir(t *testing.T) {
}
}
type duplicateReaddirFS struct {
fs.FS
dir string
names []string
}
func (d *duplicateReaddirFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
f, err := d.FS.OpenFile(name, flag, metadataOnly)
if err != nil {
return nil, err
}
if name == d.dir {
return &duplicateReaddirFile{File: f, names: d.names}, nil
}
return f, nil
}
type duplicateReaddirFile struct {
fs.File
names []string
}
func (f *duplicateReaddirFile) Readdirnames(int) ([]string, error) {
return append([]string(nil), f.names...), nil
}
func TestArchiverSaveDirDuplicateExcludedEntry(t *testing.T) {
const targetNodeName = "targetdir"
src := TestDir{
"excluded": TestFile{Content: "skip me"},
"keep": TestFile{Content: "keep me"},
}
tempdir, repo := prepareTempdirRepoSrc(t, src)
testFS := fs.Track{FS: &duplicateReaddirFS{
FS: &fs.Local{},
dir: ".",
names: []string{"excluded", "excluded", "keep"},
}}
arch := New(repo, testFS, Options{})
arch.summary = &Summary{}
arch.Select = func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
return filepath.Base(item) != "excluded"
}
arch.Error = func(item string, err error) error {
t.Errorf("unexpected archiver error for %v: %v", item, err)
return err
}
back := rtest.Chdir(t, tempdir)
defer back()
// duplicate node check in tree finder is only done if the previous tree is not nil
previousTree, err := data.NewTreeNodeIterator(strings.NewReader(`{"nodes":[]}`))
rtest.OK(t, err)
var treeID restic.ID
err = repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
meta, err := testFS.OpenFile(".", fs.O_NOFOLLOW, true)
rtest.OK(t, err)
ft, err := arch.saveDir(ctx, "/", ".", meta, previousTree, nil)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fnr := ft.take(ctx)
node := fnr.node
node.Name = targetNodeName
treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{node})
arch.stopWorkers()
return wg.Wait()
})
rtest.OK(t, err)
TestEnsureTree(context.TODO(), t, "/", repo, treeID, TestDir{
"targetdir": TestDir{
"keep": TestFile{Content: "keep me"},
},
})
}
func TestArchiverSaveDirIncremental(t *testing.T) {
tempdir := rtest.TempDir(t)
+22 -9
View File
@@ -31,9 +31,11 @@ import (
// Backend stores data on an azure endpoint.
type Backend struct {
cfg Config
container *azContainer.Client
connections uint
cfg Config
container *azContainer.Client
connections uint
prefix string
listMaxItems int
layout.Layout
accessTier blob.AccessTier
@@ -143,11 +145,12 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
be := &Backend{
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
accessTier: accessTier,
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
accessTier: accessTier,
}
return be, nil
@@ -192,6 +195,11 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
return bloberror.HasCode(err, bloberror.BlobNotFound)
@@ -223,6 +231,11 @@ func (be *Backend) Hasher() hash.Hash {
return md5.New()
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.prefix
}
// useAccessTier determines whether to apply the configured access tier to a given file.
// For archive access tier, only data files are stored using that class; metadata
// must remain instantly accessible.
@@ -406,7 +419,7 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
prefix += "/"
}
maxI := int32(defaultListMaxItems)
maxI := int32(be.listMaxItems)
opts := &azContainer.ListBlobsFlatOptions{
MaxResults: &maxI,
+22 -13
View File
@@ -23,9 +23,10 @@ import (
// b2Backend is a backend which stores its data on Backblaze B2.
type b2Backend struct {
client *b2.Client
bucket *b2.Bucket
cfg Config
client *b2.Client
bucket *b2.Bucket
cfg Config
listMaxItems int
layout.Layout
canDelete bool
@@ -106,11 +107,12 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string,
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
canDelete: true,
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
canDelete: true,
}
return be, nil
@@ -138,14 +140,20 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
}
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *b2Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
func (be *b2Backend) Properties() backend.Properties {
return backend.Properties{
Connections: be.cfg.Connections,
@@ -238,6 +246,7 @@ func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewi
// sanity check
if n != rd.Length() {
_ = w.Close()
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
}
return errors.Wrap(w.Close(), "Close")
@@ -296,7 +305,7 @@ func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backe
defer cancel()
prefix, _ := be.Basedir(t)
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(defaultListMaxItems))
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems))
for iter.Next() {
obj := iter.Object()
+26 -18
View File
@@ -7,10 +7,11 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// cacheBackend wraps a restic.cacheBackend and adds a cache.
type cacheBackend struct {
// Backend wraps a restic.Backend and adds a cache.
type Backend struct {
backend.Backend
*Cache
@@ -23,10 +24,10 @@ type cacheBackend struct {
}
// ensure Backend implements backend.Backend
var _ backend.Backend = &cacheBackend{}
var _ backend.Backend = &Backend{}
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *cacheBackend {
return &cacheBackend{
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *Backend {
return &Backend{
Backend: be,
Cache: c,
inProgress: make(map[backend.Handle]chan struct{}),
@@ -35,7 +36,7 @@ func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface
}
// Remove deletes a file from the backend and the cache if it has been cached.
func (b *cacheBackend) Remove(ctx context.Context, h backend.Handle) error {
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("cache Remove(%v)", h)
err := b.Backend.Remove(ctx, h)
if err != nil {
@@ -57,7 +58,7 @@ func autoCacheTypes(h backend.Handle) bool {
}
// Save stores a new file in the backend and the cache.
func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if !autoCacheTypes(h) {
return b.Backend.Save(ctx, h, rd)
}
@@ -91,7 +92,7 @@ func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.Re
return nil
}
func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
finish := make(chan struct{})
b.inProgressMutex.Lock()
@@ -135,7 +136,7 @@ func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
}
// loadFromCache will try to load the file from the cache.
func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
rd, inCache, err := b.Cache.load(h, length, offset)
if err != nil {
return inCache, err
@@ -150,7 +151,7 @@ func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64,
}
// Load loads a file from the cache or the backend.
func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
b.inProgressMutex.Lock()
waitForFinish, inProgress := b.inProgress[h]
b.inProgressMutex.Unlock()
@@ -197,7 +198,7 @@ func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, o
// Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache.
func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("cache Stat(%v)", h)
fi, err := b.Backend.Stat(ctx, h)
@@ -210,24 +211,31 @@ func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.File
}
// IsNotExist returns true if the error is caused by a non-existing file.
func (b *cacheBackend) IsNotExist(err error) bool {
func (b *Backend) IsNotExist(err error) bool {
return b.Backend.IsNotExist(err)
}
func (b *cacheBackend) Unwrap() backend.Backend {
func (b *Backend) Unwrap() backend.Backend {
return b.Backend
}
func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
if !b.Cache.canBeCached(t) {
return b.Backend.List(ctx, t, fn)
}
ids := make(map[string]struct{})
// will contain the IDs of the files that are in the repository
ids := restic.NewIDSet()
// wrap the original function to also add the file to the ids set
wrapFn := func(f backend.FileInfo) error {
ids[f.Name] = struct{}{}
id, err := restic.ParseID(f.Name)
if err != nil {
// ignore files with invalid name
return nil
}
ids.Insert(id)
// execute the original function
return fn(f)
@@ -252,11 +260,11 @@ func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f b
}
// Warmup delegates to wrapped backend.
func (b *cacheBackend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
return b.Backend.Warmup(ctx, h)
}
// WarmupWait delegates to wrapped backend.
func (b *cacheBackend) WarmupWait(ctx context.Context, h []backend.Handle) error {
func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error {
return b.Backend.WarmupWait(ctx, h)
}
+5 -4
View File
@@ -12,6 +12,7 @@ import (
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Cache manages a local cache.
@@ -42,10 +43,10 @@ func readVersion(dir string) (v uint, err error) {
const cacheVersion = 1
var cacheLayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshots",
backend.IndexFile: "index",
var cacheLayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshots",
restic.IndexFile: "index",
}
const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"
+12 -7
View File
@@ -12,6 +12,7 @@ import (
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
func (c *Cache) filename(h backend.Handle) string {
@@ -170,7 +171,7 @@ func (c *Cache) remove(h backend.Handle) (bool, error) {
// Clear removes all files of type t from the cache that are not contained in
// the set valid.
func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
debug.Log("Clearing cache for %v: %v valid files", t, len(valid))
if !c.canBeCached(t) {
return nil
@@ -182,12 +183,12 @@ func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
}
for id := range list {
if _, ok := valid[id]; ok {
if valid.Has(id) {
continue
}
// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id})); err != nil && !errors.Is(err, os.ErrNotExist) {
if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
@@ -200,12 +201,12 @@ func isFile(fi os.FileInfo) bool {
}
// list returns a list of all files of type T in the cache.
func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
if !c.canBeCached(t) {
return nil, errors.New("cannot be cached")
}
list := make(map[string]struct{})
list := restic.NewIDSet()
dir := filepath.Join(c.path, cacheLayoutPaths[t])
err := filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
if err != nil {
@@ -220,8 +221,12 @@ func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
return nil
}
id := filepath.Base(name)
list[id] = struct{}{}
id, err := restic.ParseID(filepath.Base(name))
if err != nil {
return nil
}
list.Insert(id)
return nil
})
+18 -19
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"fmt"
"io"
"maps"
"math/rand"
"os"
"runtime"
@@ -19,8 +18,8 @@ import (
"golang.org/x/sync/errgroup"
)
func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} {
ids := make(map[string]struct{})
func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet {
ids := restic.NewIDSet()
for i := 0; i < random.Intn(15)+10; i++ {
buf := rtest.Random(random.Int(), 1<<19)
id := restic.Hash(buf)
@@ -34,13 +33,13 @@ func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType,
if err != nil {
t.Fatal(err)
}
ids[id.String()] = struct{}{}
ids.Insert(id)
}
return ids
}
// randomID returns a random ID from s.
func randomID(s map[string]struct{}) string {
func randomID(s restic.IDSet) restic.ID {
for id := range s {
return id
}
@@ -70,7 +69,7 @@ func load(t testing.TB, c *Cache, h backend.Handle) []byte {
return buf
}
func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{} {
func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
list, err := c.list(tpe)
if err != nil {
t.Errorf("listing failed: %v", err)
@@ -79,7 +78,7 @@ func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{}
return list
}
func clearFiles(t testing.TB, c *Cache, tpe backend.FileType, valid map[string]struct{}) {
func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
if err := c.Clear(tpe, valid); err != nil {
t.Error(err)
}
@@ -103,34 +102,34 @@ func TestFiles(t *testing.T) {
ids := generateRandomFiles(t, random, tpe, c)
id := randomID(ids)
h := backend.Handle{Type: tpe, Name: id}
h := backend.Handle{Type: tpe, Name: id.String()}
id2 := restic.Hash(load(t, c, h))
if id != id2.String() {
t.Errorf("wrong data returned, want %v, got %v", id, id2.String())
if !id.Equal(id2) {
t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
}
if !c.Has(h) {
t.Errorf("cache thinks index %v isn't present", id)
t.Errorf("cache thinks index %v isn't present", id.Str())
}
list := listFiles(t, c, tpe)
if !maps.Equal(ids, list) {
if !ids.Equals(list) {
t.Errorf("wrong list of index IDs returned, want:\n %v\ngot:\n %v", ids, list)
}
clearFiles(t, c, tpe, map[string]struct{}{id: {}})
clearFiles(t, c, tpe, restic.NewIDSet(id))
list2 := listFiles(t, c, tpe)
delete(ids, id)
want := map[string]struct{}{id: {}}
if !maps.Equal(list2, want) {
ids.Delete(id)
want := restic.NewIDSet(id)
if !list2.Equals(want) {
t.Errorf("ClearIndexes removed indexes, want:\n %v\ngot:\n %v", list2, want)
}
clearFiles(t, c, tpe, map[string]struct{}{})
want = map[string]struct{}{}
clearFiles(t, c, tpe, restic.NewIDSet())
want = restic.NewIDSet()
list3 := listFiles(t, c, tpe)
if !maps.Equal(list3, want) {
if !list3.Equals(want) {
t.Errorf("ClearIndexes returned a wrong list, want:\n %v\ngot:\n %v", want, list3)
}
})
+2 -3
View File
@@ -3,17 +3,16 @@ package cache
import (
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
const testCacheID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// TestNewCache returns a cache in a temporary directory which is removed when
// cleanup is called.
func TestNewCache(t testing.TB) *Cache {
dir := test.TempDir(t)
t.Logf("created new cache at %v", dir)
cache, err := New(testCacheID, dir)
cache, err := New(restic.NewRandomID().String(), dir)
if err != nil {
t.Fatal(err)
}
+49 -33
View File
@@ -27,25 +27,27 @@ import (
"google.golang.org/api/option"
)
// gs stores data in a GCS bucket.
// Backend stores data in a GCS bucket.
//
// The service account used to access the bucket must have these permissions:
// - storage.objects.create
// - storage.objects.delete
// - storage.objects.get
// - storage.objects.list
type gs struct {
gcsClient *storage.Client
projectID string
connections uint
bucketName string
region string
bucket *storage.BucketHandle
type Backend struct {
gcsClient *storage.Client
projectID string
connections uint
bucketName string
region string
bucket *storage.BucketHandle
prefix string
listMaxItems int
layout.Layout
}
// Ensure that *Backend implements backend.Backend.
var _ backend.Backend = &gs{}
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@@ -84,7 +86,7 @@ func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
return gcsClient, nil
}
func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
_, err := bucket.Attrs(ctx)
if err == storage.ErrBucketNotExist {
return false, nil
@@ -92,7 +94,9 @@ func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (b
return err == nil, err
}
func open(cfg Config, rt http.RoundTripper) (*gs, error) {
const defaultListMaxItems = 1000
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
gcsClient, err := getStorageClient(rt)
@@ -100,14 +104,16 @@ func open(cfg Config, rt http.RoundTripper) (*gs, error) {
return nil, errors.Wrap(err, "getStorageClient")
}
be := &gs{
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
be := &Backend{
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
prefix: cfg.Prefix,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
}
return be, nil
@@ -155,12 +161,17 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *gs) IsNotExist(err error) bool {
func (be *Backend) IsNotExist(err error) bool {
return errors.Is(err, storage.ErrObjectNotExist)
}
func (be *gs) IsPermanentError(err error) bool {
func (be *Backend) IsPermanentError(err error) bool {
if be.IsNotExist(err) {
return true
}
@@ -175,7 +186,7 @@ func (be *gs) IsPermanentError(err error) bool {
return false
}
func (be *gs) Properties() backend.Properties {
func (be *Backend) Properties() backend.Properties {
return backend.Properties{
Connections: be.connections,
HasAtomicReplace: true,
@@ -183,12 +194,17 @@ func (be *gs) Properties() backend.Properties {
}
// Hasher may return a hash function for calculating a content hash for the backend
func (be *gs) Hasher() hash.Hash {
func (be *Backend) Hasher() hash.Hash {
return md5.New()
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.prefix
}
// Save stores data in the backend at the handle.
func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
// Set chunk size to zero to disable resumable uploads.
@@ -238,14 +254,14 @@ func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReade
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *gs) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length == 0 {
// negative length indicates read till end to GCS lib
length = -1
@@ -267,7 +283,7 @@ func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offs
}
// Stat returns information about a blob.
func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
attr, err := be.bucket.Object(objName).Attrs(ctx)
@@ -280,7 +296,7 @@ func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo,
}
// Remove removes the blob with the given name and type.
func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.bucket.Object(objName).Delete(ctx)
@@ -294,7 +310,7 @@ func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@@ -339,15 +355,15 @@ func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.File
}
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *gs) Delete(ctx context.Context) error {
func (be *Backend) Delete(ctx context.Context) error {
return util.DefaultDelete(ctx, be)
}
// Close does nothing.
func (be *gs) Close() error { return nil }
func (be *Backend) Close() error { return nil }
// Warmup not implemented
func (be *gs) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
return []backend.Handle{}, nil
}
func (be *gs) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
+8 -8
View File
@@ -28,8 +28,8 @@ import (
"golang.org/x/net/http2"
)
// rclone is used to access data stored somewhere via rclone.
type rclone struct {
// Backend is used to access data stored somewhere via rclone.
type Backend struct {
*rest.Backend
tr *http2.Transport
cmd *exec.Cmd
@@ -141,7 +141,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn {
}
// New initializes a Backend and starts the process.
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*rclone, error) {
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
var (
args []string
err error
@@ -196,7 +196,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
}
cmd := stdioConn.cmd
be := &rclone{
be := &Backend{
tr: tr,
cmd: cmd,
waitCh: waitCh,
@@ -270,7 +270,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
}
// Open starts an rclone process with the given config.
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
be, err := newBackend(ctx, cfg, lim, errorLog)
if err != nil {
return nil, err
@@ -297,7 +297,7 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(st
}
// Create initializes a new restic repo with rclone.
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
be, err := newBackend(ctx, cfg, lim, errorLog)
if err != nil {
return nil, err
@@ -328,7 +328,7 @@ func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(
const waitForExit = 5 * time.Second
// Close terminates the backend.
func (be *rclone) Close() error {
func (be *Backend) Close() error {
debug.Log("exiting rclone")
be.tr.CloseIdleConnections()
@@ -348,7 +348,7 @@ func (be *rclone) Close() error {
return be.waitResult
}
func (be *rclone) Properties() backend.Properties {
func (be *Backend) Properties() backend.Properties {
properties := be.Backend.Properties()
properties.HasFlakyErrors = true
return properties
+1 -1
View File
@@ -27,7 +27,7 @@ func TestRcloneExit(t *testing.T) {
_ = be.Close()
}()
err = be.(*rclone).cmd.Process.Kill()
err = be.cmd.Process.Kill()
rtest.OK(t, err)
t.Log("killed rclone")
+1
View File
@@ -245,6 +245,7 @@ func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int,
}
if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 && resp.ContentLength != int64(length) {
_ = drainAndClose(resp)
return nil, &restError{h, http.StatusRequestedRangeNotSatisfiable, "partial out of bounds read"}
}
+28 -23
View File
@@ -25,15 +25,15 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
)
// s3 stores data on an S3 endpoint.
type s3 struct {
// Backend stores data on an S3 endpoint.
type Backend struct {
client *minio.Client
cfg Config
layout.Layout
}
// make sure that *Backend implements backend.Backend
var _ backend.Backend = &s3{}
var _ backend.Backend = &Backend{}
var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"}
@@ -50,7 +50,7 @@ func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
}
func open(cfg Config, rt http.RoundTripper) (*s3, error) {
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) {
@@ -89,7 +89,7 @@ func open(cfg Config, rt http.RoundTripper) (*s3, error) {
return nil, errors.Wrap(err, "minio.New")
}
be := &s3{
be := &Backend{
client: client,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
@@ -240,12 +240,12 @@ func isAccessDenied(err error) bool {
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *s3) IsNotExist(err error) bool {
func (be *Backend) IsNotExist(err error) bool {
var e minio.ErrorResponse
return errors.As(err, &e) && e.Code == "NoSuchKey"
}
func (be *s3) IsPermanentError(err error) bool {
func (be *Backend) IsPermanentError(err error) bool {
if be.IsNotExist(err) {
return true
}
@@ -260,7 +260,7 @@ func (be *s3) IsPermanentError(err error) bool {
return false
}
func (be *s3) Properties() backend.Properties {
func (be *Backend) Properties() backend.Properties {
return backend.Properties{
Connections: be.cfg.Connections,
HasAtomicReplace: true,
@@ -268,21 +268,26 @@ func (be *s3) Properties() backend.Properties {
}
// Hasher may return a hash function for calculating a content hash for the backend
func (be *s3) Hasher() hash.Hash {
func (be *Backend) Hasher() hash.Hash {
return nil
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.cfg.Prefix
}
// useStorageClass returns whether file should be saved in the provided Storage Class
// For archive storage classes, only data files are stored using that class; metadata
// must remain instantly accessible.
func (be *s3) useStorageClass(h backend.Handle) bool {
func (be *Backend) useStorageClass(h backend.Handle) bool {
isDataFile := h.Type == backend.PackFile && !h.IsMetadata
isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass)
return !isArchiveClass || isDataFile
}
// Save stores data in the backend at the handle.
func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
opts := minio.PutObjectOptions{
@@ -308,14 +313,14 @@ func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReade
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *s3) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
opts := minio.GetObjectOptions{}
@@ -347,7 +352,7 @@ func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offs
}
// Stat returns information about a blob.
func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
var obj *minio.Object
@@ -375,7 +380,7 @@ func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo,
}
// Remove removes the blob with the given name and type.
func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@@ -389,7 +394,7 @@ func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, recursive := be.Basedir(t)
// make sure prefix ends with a slash
@@ -444,15 +449,15 @@ func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.File
}
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *s3) Delete(ctx context.Context) error {
func (be *Backend) Delete(ctx context.Context) error {
return util.DefaultDelete(ctx, be)
}
// Close does nothing
func (be *s3) Close() error { return nil }
func (be *Backend) Close() error { return nil }
// Warmup transitions handles from cold to hot storage if needed.
func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
handlesWarmingUp := []backend.Handle{}
if be.cfg.EnableRestore {
@@ -473,7 +478,7 @@ func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.H
}
// requestRestore sends a glacier restore request on a given file.
func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error) {
func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) {
objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{})
if err != nil {
return false, err
@@ -509,7 +514,7 @@ func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error)
}
// getWarmupStatus returns the warmup status of the provided object.
func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
// We can't use objectInfo.StorageClass to get the storage class of the
// object because this field is only set during ListObjects operations.
// The response header is the documented way to get the storage class
@@ -540,7 +545,7 @@ func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
}
// WarmupWait waits until all handles are in hot storage.
func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error {
timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout)
defer timeoutCtxCancel()
@@ -559,7 +564,7 @@ func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
}
// waitForRestore waits for a given file to be restored.
func (be *s3) waitForRestore(ctx context.Context, filename string) error {
func (be *Backend) waitForRestore(ctx context.Context, filename string) error {
for {
var objectInfo minio.ObjectInfo
+38 -6
View File
@@ -11,6 +11,7 @@ import (
"os"
"os/exec"
"path"
"sync/atomic"
"syscall"
"time"
@@ -37,7 +38,8 @@ type SFTP struct {
cmd *exec.Cmd
result <-chan error
posixRename bool
posixRename bool
chmodBeforeRemove atomic.Bool
layout.Layout
Config
@@ -363,6 +365,7 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
if err == nil {
err = f.Chmod(r.Modes.File)
if err != nil {
_ = f.Close()
return errors.Wrapf(err, "Chmod %v", tmpFilename)
}
}
@@ -404,12 +407,11 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
} else {
err = r.c.Rename(tmpFilename, filename)
}
err = setFileReadonly(r.c, filename, r.Modes.File)
if err != nil {
return errors.Errorf("sftp setFileReadonly: %v", err)
return errors.Wrapf(err, "Rename %v", tmpFilename)
}
return errors.Wrapf(err, "Rename %v", tmpFilename)
err = setFileReadonly(r.c, filename, r.Modes.File)
return errors.Wrapf(err, "setFileReadonly %v", filename)
}
// checkNoSpace checks if err was likely caused by lack of available space
@@ -507,7 +509,37 @@ func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
return err
}
return errors.Wrapf(r.c.Remove(r.Filename(h)), "Remove %v", r.Filename(h))
path := r.Filename(h)
if r.chmodBeforeRemove.Load() {
return r.removeWithChmod(path)
}
// optimistically try to remove the file
err := r.c.Remove(path)
if err == nil {
return nil
}
if !errors.Is(err, os.ErrPermission) {
return errors.Wrapf(err, "Remove %v", path)
}
// fallback to chmod + remove
// this is necessary on Windows where read-only files cannot be deleted without chmod.
if err := r.removeWithChmod(path); err != nil {
return err
}
r.chmodBeforeRemove.Store(true)
return nil
}
func (r *SFTP) removeWithChmod(path string) error {
err := r.c.Chmod(path, r.Modes.File)
if err != nil {
return errors.Wrapf(err, "Chmod %v", path)
}
return errors.Wrapf(r.c.Remove(path), "Remove %v", path)
}
// List runs fn for each file in the backend which has the type t. When an
+47 -28
View File
@@ -251,6 +251,10 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
test.OK(t, b.Remove(context.TODO(), handle))
}
type setter interface {
SetListMaxItems(int)
}
// TestList makes sure that the backend implements List() pagination correctly.
func (s *Suite[C]) TestList(t *testing.T) {
random := seedRand(t)
@@ -298,39 +302,54 @@ func (s *Suite[C]) TestList(t *testing.T) {
t.Logf("wrote %v files", len(list1))
list2 := make(map[restic.ID]int64)
err = b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
}
list2[id] = fi.Size
return nil
})
if err != nil {
t.Fatalf("List returned error %v", err)
var tests = []struct {
maxItems int
}{
{11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123},
}
t.Logf("loaded %v IDs from backend", len(list2))
for _, test := range tests {
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
list2 := make(map[restic.ID]int64)
for id, size := range list1 {
size2, ok := list2[id]
if !ok {
t.Errorf("id %v not returned by List()", id.Str())
}
if s, ok := b.(setter); ok {
t.Logf("setting max list items to %d", test.maxItems)
s.SetListMaxItems(test.maxItems)
}
if size != size2 {
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
}
}
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
}
list2[id] = fi.Size
return nil
})
for id := range list2 {
_, ok := list1[id]
if !ok {
t.Errorf("extra id %v returned by List()", id.Str())
}
if err != nil {
t.Fatalf("List returned error %v", err)
}
t.Logf("loaded %v IDs from backend", len(list2))
for id, size := range list1 {
size2, ok := list2[id]
if !ok {
t.Errorf("id %v not returned by List()", id.Str())
}
if size != size2 {
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
}
}
for id := range list2 {
_, ok := list1[id]
if !ok {
t.Errorf("extra id %v returned by List()", id.Str())
}
}
})
}
t.Logf("remove %d files", numTestFiles)
+7 -2
View File
@@ -35,10 +35,15 @@ func DefaultDelete(ctx context.Context, be backend.Backend) error {
for _, t := range alltypes {
err := be.List(ctx, t, func(fi backend.FileInfo) error {
return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
err := be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
if err != nil && be.IsNotExist(err) {
// deletion of files created by TestSaveError may happen with a delay for the REST server, so we ignore the error
err = nil
}
return err
})
if err != nil {
return nil
return err
}
}
err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})
+59 -30
View File
@@ -4,6 +4,7 @@ import (
"context"
"io"
"math/rand"
"os"
"path/filepath"
"sort"
"strconv"
@@ -18,6 +19,7 @@ import (
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/hashing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@@ -73,8 +75,7 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) {
}
func TestCheckRepo(t *testing.T) {
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
repo, _ := repository.TestFromFixture(t, checkerTestData)
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -91,8 +92,7 @@ func TestCheckRepo(t *testing.T) {
}
func TestMissingPack(t *testing.T) {
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
repo, be := repository.TestFromFixture(t, checkerTestData)
packID := restic.TestParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6")
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: packID.String()}))
@@ -109,7 +109,7 @@ func TestMissingPack(t *testing.T) {
test.Assert(t, len(errs) == 1,
"expected exactly one error, got %v", len(errs))
if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
if err, ok := errs[0].(*repository.PackError); ok {
test.Equals(t, packID, err.ID)
} else {
t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -117,8 +117,7 @@ func TestMissingPack(t *testing.T) {
}
func TestUnreferencedPack(t *testing.T) {
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
repo, be := repository.TestFromFixture(t, checkerTestData)
// index 3f1a only references pack 60e0
packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
@@ -137,7 +136,7 @@ func TestUnreferencedPack(t *testing.T) {
test.Assert(t, len(errs) == 1,
"expected exactly one error, got %v", len(errs))
if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
if err, ok := errs[0].(*repository.PackError); ok {
test.Equals(t, packID, err.ID.String())
} else {
t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -145,8 +144,7 @@ func TestUnreferencedPack(t *testing.T) {
}
func TestUnreferencedBlobs(t *testing.T) {
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
repo, be := repository.TestFromFixture(t, checkerTestData)
snapshotID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.SnapshotFile, Name: snapshotID.String()}))
@@ -180,8 +178,7 @@ func TestUnreferencedBlobs(t *testing.T) {
}
func TestModifiedIndex(t *testing.T) {
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
repo, be := repository.TestFromFixture(t, checkerTestData)
done := make(chan struct{})
defer close(done)
@@ -190,19 +187,56 @@ func TestModifiedIndex(t *testing.T) {
Type: restic.IndexFile,
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
var data []byte
test.OK(t, be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
var err error
data, err = io.ReadAll(rd)
tmpfile, err := os.CreateTemp("", "restic-test-mod-index-")
if err != nil {
t.Fatal(err)
}
defer func() {
err := tmpfile.Close()
if err != nil {
t.Fatal(err)
}
err = os.Remove(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
}()
wr := io.Writer(tmpfile)
var hw *hashing.Writer
if be.Hasher() != nil {
hw = hashing.NewWriter(wr, be.Hasher())
wr = hw
}
// read the file from the backend
err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
_, err := io.Copy(wr, rd)
return err
}))
})
test.OK(t, err)
// save the index again with a modified name so that the hash doesn't match
// the content any more
h2 := backend.Handle{
Type: restic.IndexFile,
Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher())))
var hash []byte
if hw != nil {
hash = hw.Sum(nil)
}
rd, err := backend.NewFileReader(tmpfile, hash)
if err != nil {
t.Fatal(err)
}
err = be.Save(context.TODO(), h2, rd)
if err != nil {
t.Fatal(err)
}
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -220,8 +254,7 @@ func TestModifiedIndex(t *testing.T) {
var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
func TestDuplicatePacksInIndex(t *testing.T) {
repo, _, cleanup := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
defer cleanup()
repo, _ := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -420,8 +453,7 @@ func (r *loadTreesOnceRepository) LoadBlob(ctx context.Context, t restic.BlobTyp
}
func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
repo, _ := repository.TestFromFixture(t, checkerTestData)
checkRepo := &loadTreesOnceRepository{
Repository: repo,
loadedTrees: restic.NewIDSet(),
@@ -569,13 +601,12 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
test.Assert(t, delayRepo.Triggered, "delay repository did not trigger")
}
func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, func()) {
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository) {
repo, _ := repository.TestFromFixture(t, checkerTestData)
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
if len(errs) > 0 {
defer cleanup()
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
}
@@ -584,12 +615,11 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun
t.Fatalf("expected mixed pack hint, got %v", err)
}
}
return chkr, repo, cleanup
return chkr, repo
}
func BenchmarkChecker(t *testing.B) {
chkr, _, cleanup := loadBenchRepository(t)
defer cleanup()
chkr, _ := loadBenchRepository(t)
t.ResetTimer()
@@ -601,8 +631,7 @@ func BenchmarkChecker(t *testing.B) {
}
func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) {
chkr, repo, cleanup := loadBenchRepository(t)
defer cleanup()
chkr, repo := loadBenchRepository(t)
snID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
sn2, err := data.LoadSnapshot(context.TODO(), repo, snID)
+1 -1
View File
@@ -109,7 +109,7 @@ func nodeFillExtendedAttributes(node *data.Node, path string, ignoreListError bo
for _, attr := range xattrs {
attrVal, err := getxattr(path, attr)
if err != nil {
warnf("can not obtain extended attribute %v for %v:\n", attr, path)
warnf("can not obtain extended attribute %v for %v: %v\n", attr, path, err)
continue
}
attr := data.ExtendedAttribute{
+1 -1
View File
@@ -225,7 +225,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
return nil, err
}
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
return d.cache.lookupOrCreate(name, -1, func(forget forgetFn) (fs.Node, error) {
node, ok := d.items[name]
if !ok {
debug.Log(" Lookup(%v) -> not found", name)
+33
View File
@@ -247,6 +247,39 @@ func TestStableNodeObjects(t *testing.T) {
testStableLookup(t, dir, "file-2")
}
func TestSnapshotsDirLatestSymlinkUpdatesAfterReload(t *testing.T) {
repo := repository.TestRepository(t)
timeTemplate := "2006-01-02T15:04:05"
firstTime, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56Z")
rtest.OK(t, err)
secondTime := firstTime.Add(time.Hour)
data.TestCreateSnapshot(t, repo, firstTime, 0)
root := NewRoot(repo, Config{
TimeTemplate: timeTemplate,
PathTemplates: []string{"snapshots/%T"},
})
snapshotsDir := testStableLookup(t, root, "snapshots")
rtest.Equals(t, firstTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
data.TestCreateSnapshot(t, repo, secondTime, 0)
root.SnapshotsDir.dirStruct.lastCheck = time.Now().Add(-2 * minSnapshotsReloadTime)
rtest.Equals(t, secondTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
}
func readLatestTarget(t testing.TB, node fs.Node) string {
t.Helper()
latest, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), "latest")
rtest.OK(t, err)
target, err := latest.(fs.NodeReadlinker).Readlink(context.TODO(), nil)
rtest.OK(t, err)
return target
}
// Test reporting of fuse.Attr.Blocks in multiples of 512.
func TestBlocks(t *testing.T) {
root := &Root{}
+3 -3
View File
@@ -61,7 +61,7 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("ReadDirAll()")
// update snapshots
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
meta, _, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
return nil, unwrapCtxCanceled(err)
} else if meta == nil {
@@ -104,14 +104,14 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("Lookup(%s)", name)
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
meta, gen, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
return nil, unwrapCtxCanceled(err)
} else if meta == nil {
return nil, syscall.ENOENT
}
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
return d.cache.lookupOrCreate(name, gen, func(forget forgetFn) (fs.Node, error) {
entry := meta.names[name]
if entry == nil {
return nil, syscall.ENOENT
+8 -3
View File
@@ -42,6 +42,10 @@ type SnapshotsDirStructure struct {
hash [sha256.Size]byte // Hash at last check.
lastCheck time.Time
// generation is incremented whenever the directory structure is rebuilt.
// It allows treeCache instances to detect stale entries and reset themselves.
generation int64
}
// NewSnapshotsDirStructure returns a new directory structure for snapshots.
@@ -280,6 +284,7 @@ func (d *SnapshotsDirStructure) makeDirs(snapshots data.Snapshots) {
}
d.entries = entries
d.generation++
}
const minSnapshotsReloadTime = 60 * time.Second
@@ -337,13 +342,13 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
return nil
}
func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) {
func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, int64, error) {
err := d.updateSnapshots(ctx)
if err != nil {
return nil, err
return nil, 0, err
}
d.mutex.Lock()
defer d.mutex.Unlock()
return d.entries[prefix], nil
return d.entries[prefix], d.generation, nil
}
+16 -4
View File
@@ -5,12 +5,15 @@ package fuse
import (
"sync"
"github.com/restic/restic/internal/debug"
"github.com/anacrolix/fuse/fs"
)
type treeCache struct {
nodes map[string]fs.Node
m sync.Mutex
nodes map[string]fs.Node
m sync.Mutex
generation int64
}
type forgetFn func()
@@ -21,19 +24,28 @@ func newTreeCache() *treeCache {
}
}
func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
func (t *treeCache) lookupOrCreate(name string, generation int64, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
t.m.Lock()
defer t.m.Unlock()
if generation >= 0 && generation != t.generation {
debug.Log("treeCache generation changed %d -> %d, resetting cache", t.generation, generation)
t.nodes = make(map[string]fs.Node)
t.generation = generation
}
if node, ok := t.nodes[name]; ok {
return node, nil
}
cacheGeneration := t.generation
node, err := create(func() {
t.m.Lock()
defer t.m.Unlock()
delete(t.nodes, name)
if t.generation == cacheGeneration {
delete(t.nodes, name)
}
})
if err != nil {
return nil, err
+82
View File
@@ -0,0 +1,82 @@
//go:build darwin || freebsd || linux
package fuse
import (
"context"
"testing"
"github.com/anacrolix/fuse"
"github.com/anacrolix/fuse/fs"
"github.com/restic/restic/internal/test"
)
type cacheTestNode struct {
id int
}
func (n cacheTestNode) Attr(context.Context, *fuse.Attr) error {
return nil
}
func TestTreeCacheGeneration(t *testing.T) {
cache := newTreeCache()
created := 0
create := func(forgetFn) (fs.Node, error) {
created++
return cacheTestNode{id: created}, nil
}
node1, err := cache.lookupOrCreate("node", 1, create)
test.OK(t, err)
node2, err := cache.lookupOrCreate("node", 1, create)
test.OK(t, err)
test.Assert(t, node1 == node2, "lookup should reuse cached node")
test.Equals(t, 1, created)
node3, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node1 != node3, "lookup should recreate node after generation change")
test.Equals(t, 2, created)
node4, err := cache.lookupOrCreate("node", -1, create)
test.OK(t, err)
test.Assert(t, node3 == node4, "negative generation should not reset cache")
test.Equals(t, 2, created)
node5, err := cache.lookupOrCreate("node", 3, create)
test.OK(t, err)
test.Assert(t, node4 != node5, "lookup should still track later generation changes")
test.Equals(t, 3, created)
}
func TestTreeCacheForgetOnlyRemovesSameGeneration(t *testing.T) {
cache := newTreeCache()
created := 0
var forgets []forgetFn
create := func(forget forgetFn) (fs.Node, error) {
created++
forgets = append(forgets, forget)
return cacheTestNode{id: created}, nil
}
node1, err := cache.lookupOrCreate("node", 1, create)
test.OK(t, err)
node2, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node1 != node2, "lookup should recreate node after generation change")
test.Equals(t, 2, created)
forgets[0]()
node3, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node2 == node3, "forget from an old generation must not remove the current node")
test.Equals(t, 2, created)
forgets[1]()
node4, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node3 != node4, "forget from the current generation should remove the cached node")
test.Equals(t, 3, created)
}
+1 -1
View File
@@ -35,7 +35,7 @@ import (
// to a missing backend storage location or config file
var ErrNoRepository = errors.New("repository does not exist")
const Version = "0.19.0-dev (compiled manually)"
const Version = "0.19.1-dev (compiled manually)"
// TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05"
+215
View File
@@ -0,0 +1,215 @@
package repository
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"fmt"
"io"
"github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/hashing"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
)
// ErrPackData is returned if errors are discovered while verifying a packfile
type ErrPackData struct {
PackID restic.ID
errs []error
}
func (e *ErrPackData) Error() string {
return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
}
type partialReadError struct {
err error
}
func (e *partialReadError) Error() string {
return e.err.Error()
}
// CheckPack reads a pack and checks the integrity of all blobs.
func CheckPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err != nil {
if r.cache != nil {
// ignore error as there's not much we can do here
_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
}
// retry pack verification to detect transient errors
err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err2 != nil {
err = err2
} else {
err = fmt.Errorf("check successful on second attempt, original error %w", err)
}
}
return err
}
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
debug.Log("checking pack %v", id.String())
if len(blobs) == 0 {
return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
}
// sanity check blobs in index
blobs.Sort()
idxHdrSize := pack.CalculateHeaderSize(blobs)
lastBlobEnd := 0
nonContinuousPack := false
for _, blob := range blobs {
if lastBlobEnd != int(blob.Offset) {
nonContinuousPack = true
}
lastBlobEnd = int(blob.Offset + blob.Length)
}
// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
var errs []error
if nonContinuousPack {
debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
}
// calculate hash on-the-fly while reading the pack and capture pack header
var hash restic.ID
var hdrBuf []byte
// must use a separate slice from `errs` here as we're only interested in the last retry
var blobErrors []error
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
hrd := hashing.NewReader(rd, sha256.New())
bufRd.Reset(hrd)
// reset blob errors for each retry
blobErrors = nil
it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
for {
if ctx.Err() != nil {
return ctx.Err()
}
val, err := it.Next()
if err == errPackEOF {
break
} else if err != nil {
return &partialReadError{err}
}
debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle)
if val.Err != nil {
debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err)
blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
}
}
// skip enough bytes until we reach the possible header start
curPos := lastBlobEnd
minHdrStart := int(size) - pack.MaxHeaderSize
if minHdrStart > curPos {
_, err := bufRd.Discard(minHdrStart - curPos)
if err != nil {
return &partialReadError{err}
}
curPos += minHdrStart - curPos
}
// read remainder, which should be the pack header
var err error
hdrBuf = make([]byte, int(size-int64(curPos)))
_, err = io.ReadFull(bufRd, hdrBuf)
if err != nil {
return &partialReadError{err}
}
hash = restic.IDFromHash(hrd.Sum(nil))
return nil
})
errs = append(errs, blobErrors...)
if err != nil {
var e *partialReadError
isPartialReadError := errors.As(err, &e)
// failed to load the pack file, return as further checks cannot succeed anyways
debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err)
if isPartialReadError {
return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
}
// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
// completely failed to download such that there's no point in repairing anything.
return fmt.Errorf("download error: %w", err)
}
if !hash.Equal(id) {
debug.Log("pack ID does not match, want %v, got %v", id, hash)
return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
}
blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
if err != nil {
return &ErrPackData{PackID: id, errs: append(errs, err)}
}
if uint32(idxHdrSize) != hdrSize {
debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
}
for _, blob := range blobs {
// Check if blob is contained in index and position is correct
idxHas := false
for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
if pb.PackID == id && pb.Blob == blob {
idxHas = true
break
}
}
if !idxHas {
errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
continue
}
}
if len(errs) > 0 {
return &ErrPackData{PackID: id, errs: errs}
}
return nil
}
type bufReader struct {
rd *bufio.Reader
buf []byte
}
func newBufReader(rd *bufio.Reader) *bufReader {
return &bufReader{
rd: rd,
}
}
func (b *bufReader) Discard(n int) (discarded int, err error) {
return b.rd.Discard(n)
}
func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
if cap(b.buf) < n {
b.buf = make([]byte, n)
}
b.buf = b.buf[:n]
_, err = io.ReadFull(b.rd, b.buf)
if err != nil {
return nil, err
}
return b.buf, nil
}
+8 -207
View File
@@ -2,17 +2,12 @@ package repository
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"fmt"
"io"
"github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/hashing"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
@@ -51,29 +46,18 @@ func (e *ErrMixedPack) Error() string {
return fmt.Sprintf("pack %v contains a mix of tree and data blobs", e.PackID.Str())
}
// ErrPackMetadata describes an error with a specific pack. It is used for missing, truncated or orphaned packs.
// Errors of the actual pack data are returned as ErrPackData.
type ErrPackMetadata struct {
// PackError describes an error with a specific pack.
type PackError struct {
ID restic.ID
Orphaned bool
Truncated bool
Err error
}
func (e *ErrPackMetadata) Error() string {
func (e *PackError) Error() string {
return "pack " + e.ID.String() + ": " + e.Err.Error()
}
// ErrPackData is returned if errors are discovered while verifying a packfile
type ErrPackData struct {
PackID restic.ID
errs []error
}
func (e *ErrPackData) Error() string {
return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
}
// Checker handles index-related operations for repository checking.
type Checker struct {
repo *Repository
@@ -215,7 +199,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
select {
case <-ctx.Done():
return
case errChan <- &ErrPackMetadata{ID: id, Err: errors.New("does not exist")}:
case errChan <- &PackError{ID: id, Err: errors.New("does not exist")}:
}
continue
}
@@ -225,7 +209,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
select {
case <-ctx.Done():
return
case errChan <- &ErrPackMetadata{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
}
}
}
@@ -235,7 +219,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
select {
case <-ctx.Done():
return
case errChan <- &ErrPackMetadata{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
case errChan <- &PackError{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
}
}
}
@@ -269,7 +253,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize)
dec, err := zstd.NewReader(nil)
if err != nil {
panic(dec)
panic(err)
}
defer dec.Close()
for {
@@ -285,7 +269,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
}
}
err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
err := CheckPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
p.Add(1)
if err == nil {
continue
@@ -325,186 +309,3 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
}
}
}
// checkPack reads a pack and checks the integrity of all blobs.
func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err != nil {
if r.cache != nil {
// ignore error as there's not much we can do here
_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
}
// retry pack verification to detect transient errors
err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err2 != nil {
err = err2
} else {
err = fmt.Errorf("check successful on second attempt, original error %w", err)
}
}
return err
}
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
type partialReadError struct {
error
}
debug.Log("checking pack %v", id.String())
if len(blobs) == 0 {
return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
}
// sanity check blobs in index
blobs.Sort()
idxHdrSize := pack.CalculateHeaderSize(blobs)
lastBlobEnd := 0
nonContinuousPack := false
for _, blob := range blobs {
if lastBlobEnd != int(blob.Offset) {
nonContinuousPack = true
}
lastBlobEnd = int(blob.Offset + blob.Length)
}
// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
var errs []error
if nonContinuousPack {
debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
}
// calculate hash on-the-fly while reading the pack and capture pack header
var hash restic.ID
var hdrBuf []byte
// must use a separate slice from `errs` here as we're only interested in the last retry
var blobErrors []error
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
hrd := hashing.NewReader(rd, sha256.New())
bufRd.Reset(hrd)
// reset blob errors for each retry
blobErrors = nil
it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
for {
if ctx.Err() != nil {
return ctx.Err()
}
val, err := it.Next()
if err == errPackEOF {
break
} else if err != nil {
return &partialReadError{err}
}
debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle)
if val.Err != nil {
debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err)
blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
}
}
// skip enough bytes until we reach the possible header start
curPos := lastBlobEnd
minHdrStart := int(size) - pack.MaxHeaderSize
if minHdrStart > curPos {
_, err := bufRd.Discard(minHdrStart - curPos)
if err != nil {
return &partialReadError{err}
}
curPos += minHdrStart - curPos
}
// read remainder, which should be the pack header
var err error
hdrBuf = make([]byte, int(size-int64(curPos)))
_, err = io.ReadFull(bufRd, hdrBuf)
if err != nil {
return &partialReadError{err}
}
hash = restic.IDFromHash(hrd.Sum(nil))
return nil
})
errs = append(errs, blobErrors...)
if err != nil {
var e *partialReadError
isPartialReadError := errors.As(err, &e)
// failed to load the pack file, return as further checks cannot succeed anyways
debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err)
if isPartialReadError {
return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
}
// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
// completely failed to download such that there's no point in repairing anything.
return fmt.Errorf("download error: %w", err)
}
if !hash.Equal(id) {
debug.Log("pack ID does not match, want %v, got %v", id, hash)
return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
}
blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
if err != nil {
return &ErrPackData{PackID: id, errs: append(errs, err)}
}
if uint32(idxHdrSize) != hdrSize {
debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
}
for _, blob := range blobs {
// Check if blob is contained in index and position is correct
idxHas := false
for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
if pb.PackID == id && pb.Blob == blob {
idxHas = true
break
}
}
if !idxHas {
errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
continue
}
}
if len(errs) > 0 {
return &ErrPackData{PackID: id, errs: errs}
}
return nil
}
type bufReader struct {
rd *bufio.Reader
buf []byte
}
func newBufReader(rd *bufio.Reader) *bufReader {
return &bufReader{
rd: rd,
}
}
func (b *bufReader) Discard(n int) (discarded int, err error) {
return b.rd.Discard(n)
}
func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
if cap(b.buf) < n {
b.buf = make([]byte, n)
}
b.buf = b.buf[:n]
_, err = io.ReadFull(b.rd, b.buf)
if err != nil {
return nil, err
}
return b.buf, nil
}
-390
View File
@@ -1,390 +0,0 @@
//go:build debug
package repository
import (
"context"
"crypto/aes"
"crypto/cipher"
"encoding/json"
"fmt"
"io"
"os"
"runtime"
"sync"
"time"
"github.com/klauspost/compress/zstd"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
)
type packDumpEntry struct {
Name string `json:"name"`
Blobs []packDumpBlob `json:"blobs"`
}
type packDumpBlob struct {
Type restic.BlobType `json:"type"`
Length uint `json:"length"`
ID restic.ID `json:"id"`
Offset uint `json:"offset"`
}
func writePackDumpJSON(wr io.Writer, item any) error {
buf, err := json.MarshalIndent(item, "", " ")
if err != nil {
return err
}
_, err = wr.Write(append(buf, '\n'))
return err
}
// DumpPacks lists each pack file and writes its header blob layout as JSON to wr.
func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, err := repo.ListPack(ctx, id, size)
if err != nil {
printer.E("error for pack %v: %v", id.Str(), err)
return nil
}
p := packDumpEntry{
Name: id.String(),
Blobs: make([]packDumpBlob, len(blobs)),
}
for i, blob := range blobs {
p.Blobs[i] = packDumpBlob{
Type: blob.Type,
Length: blob.Length,
ID: blob.ID,
Offset: blob.Offset,
}
}
m.Lock()
defer m.Unlock()
return writePackDumpJSON(wr, p)
})
}
// DumpIndexes loads each on-disk index file and writes its debug dump to wr.
func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id)
if err != nil {
return err
}
return idx.Dump(wr)
})
}
// ExaminePackOptions configures debug examination of a pack file.
type ExaminePackOptions struct {
TryRepair bool
RepairByte bool
ExtractPack bool
ReuploadBlobs bool
}
// ExaminePack loads and inspects a pack file and its index entries.
func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error {
printer.S("examine %v", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
printer.S(" file size is %v", len(buf))
gotID := restic.Hash(buf)
if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID)
} else {
printer.S(" hash for file content matches")
}
printer.S(" ========================================")
printer.S(" looking for info in the indexes")
blobsLoaded := false
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
continue
}
checkPackSize(blobs, len(buf), printer)
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
if err != nil {
printer.E("error: %v", err)
} else {
blobsLoaded = true
}
}
printer.S(" ========================================")
printer.S(" inspect the pack itself")
blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
checkPackSize(blobs, len(buf), printer)
if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer)
}
return nil
}
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
// track current size and offset
var size, offset uint64
blobs.Sort()
for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
}
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
}
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
} else {
printer.S(" file sizes match")
}
}
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
if bytewise {
printer.S(" trying to repair blob by finding a broken byte")
} else {
printer.S(" trying to repair blob with single bit flip")
}
ch := make(chan int)
var wg errgroup.Group
done := make(chan struct{})
var fixed []byte
var found bool
workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ {
wg.Go(func() error {
// make a local copy of the buffer
buf := make([]byte, len(input))
copy(buf, input)
testFlip := func(idx int, pattern byte) bool {
// flip bits
buf[idx] ^= pattern
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil {
printer.S("")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext))
close(done)
found = true
fixed = plaintext
return true
}
// flip bits back
buf[idx] ^= pattern
return false
}
for i := range ch {
if bytewise {
for j := 0; j < 255; j++ {
if testFlip(i, byte(j)) {
return nil
}
}
} else {
for j := 0; j < 7; j++ {
// flip each bit once
if testFlip(i, (1 << uint(j))) {
return nil
}
}
}
}
return nil
})
}
wg.Go(func() error {
defer close(ch)
start := time.Now()
info := time.Now()
for i := range input {
select {
case ch <- i:
case <-done:
printer.S(" done after %v", time.Since(start))
return nil
}
if time.Since(info) > time.Second {
secs := time.Since(start).Seconds()
gps := float64(i) / secs
remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now()
}
}
return nil
})
err := wg.Wait()
if err != nil {
panic("all go routines can only return nil")
}
if !found {
printer.S("\n blob could not be repaired")
}
return fixed
}
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
// strip signature at the end
l := len(buf)
nonce, ct := buf[:16], buf[16:l-16]
out := make([]byte, len(ct))
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCTR(c, nonce)
e.XORKeyStream(out, ct)
return out
}
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
packData, err := repo.LoadRaw(ctx, restic.PackFile, packID)
// allow processing broken pack files
if packData == nil {
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(packData) {
printer.E("skipping truncated blob")
continue
}
buf := packData[blob.Offset : blob.Offset+blob.Length]
key := repo.Key()
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
outputPrefix := ""
filePrefix := ""
if err != nil {
printer.E("error decrypting blob: %v", err)
if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
if err != nil {
return err
}
continue
}
}
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
printer.S(" failed to decompress blob %v", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
}
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
if err != nil {
return err
}
}
if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
printer.S(" uploaded %v %v", blob.Type, id)
}
}
return nil
})
return err
}
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename)
if err != nil {
return err
}
_, err = f.Write(plain)
if err != nil {
_ = f.Close()
return err
}
err = f.Close()
if err != nil {
return err
}
printer.S("decrypt of blob %v stored at %v", id, filename)
return nil
}
@@ -15,8 +15,7 @@ import (
var repoFixture = filepath.Join("..", "testdata", "test-repo.tar.gz")
func TestRepositoryForAllIndexes(t *testing.T) {
repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
defer cleanup()
repo, _ := repository.TestFromFixture(t, repoFixture)
expectedIndexIDs := restic.NewIDSet()
rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
+1 -1
View File
@@ -280,6 +280,7 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
}
func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
defer p.Done()
indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
if err != nil {
return err
@@ -302,7 +303,6 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
return err
}
p.SetMax(numIndexFiles)
defer p.Done()
}
err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error {
@@ -694,62 +694,3 @@ func TestRewriteSplitPacks(t *testing.T) {
blobs := mi.Lookup(blobOther.BlobHandle)
rtest.Equals(t, nil, blobs)
}
// TestRewriteFullPacks checks that Rewrite drops a duplicate full index for the same
// pack while keeping the other index files and blob lookups intact. Creates 3 indexes:
// - indexA: contains packA
// - indexB: contains packB
// - indexC: contains packB
// After the rewrite, indexC must be dropped. The other indexes must be kept.
func TestRewriteFullPacks(t *testing.T) {
originalFull := index.Full
defer func() {
index.Full = originalFull
}()
index.Full = func(*index.Index) bool { return true }
repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion)
packA := restic.NewRandomID()
packB := restic.NewRandomID()
blobA := restic.PackedBlob{
PackID: packA,
Blob: restic.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
blobB := restic.PackedBlob{
PackID: packB,
Blob: restic.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(50)),
Offset: 0,
},
}
mi := index.NewMasterIndex()
rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
indexIDs := mi.IDs()
rtest.Equals(t, 3, len(indexIDs))
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
mi2 := index.NewMasterIndex()
rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
afterRewrite := mi2.IDs()
rtest.Equals(t, 2, len(afterRewrite))
rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
}
-41
View File
@@ -1,41 +0,0 @@
package repository
import (
"context"
"iter"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
)
// IndexBlob is one blob handle from an on-disk index file, or an error from loading/decoding
// that file.
type IndexBlob struct {
Handle restic.BlobHandle
Error error
}
// AllIndexBlobs streams blob handles from each index file without building a master index.
func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.LoaderUnpacked) iter.Seq[IndexBlob] {
return func(yield func(IndexBlob) bool) {
stopIteration := errors.New("stop index blob iteration")
err := index.ForAllIndexes(ctx, lister, loader, func(_ restic.ID, idx *index.Index, err error) error {
if err != nil {
return err
}
for blob := range idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
}
if !yield(IndexBlob{Handle: blob.BlobHandle}) {
return stopIteration
}
}
return nil
})
if err != nil && !errors.Is(err, stopIteration) {
yield(IndexBlob{Error: err})
}
}
}
-62
View File
@@ -1,62 +0,0 @@
package repository_test
import (
"context"
"testing"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestAllIndexBlobs(t *testing.T) {
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
want := restic.NewBlobSet()
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for i := range 5 {
data := []byte{byte('a' + i)}
id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, data, restic.ID{}, false)
rtest.OK(t, err)
want.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id})
}
return nil
}))
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
fromMaster := restic.NewBlobSet()
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
fromMaster.Insert(pb.BlobHandle)
}))
rtest.Equals(t, want, fromMaster)
fromStream := restic.NewBlobSet()
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
if entry.Error != nil {
t.Fatalf("unexpected error: %v", entry.Error)
}
fromStream.Insert(entry.Handle)
}
rtest.Equals(t, want, fromStream)
}
func TestAllIndexBlobsEarlyStop(t *testing.T) {
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for range 5 {
_, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, []byte("test"), restic.ID{}, false)
rtest.OK(t, err)
}
return nil
}))
var count int
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
rtest.Assert(t, entry.Error == nil, "unexpected error after early stop: %v", entry.Error)
count++
break
}
rtest.Equals(t, 1, count)
}
+2
View File
@@ -586,6 +586,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
// unreferenced packs can be safely deleted first
if len(plan.removePacksFirst) != 0 {
printer.P("deleting unreferenced packs\n")
// ignoring errors is fine here as keeping too many packs cannot damage the repository
_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacksFirst, restic.PackFile, printer)
// forget unused data
plan.removePacksFirst = nil
@@ -643,6 +644,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
if len(plan.removePacks) != 0 {
printer.P("removing %d old packs", len(plan.removePacks))
// ignoring errors is fine here as keeping too many packs cannot damage the repository
_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacks, restic.PackFile, printer)
}
if ctx.Err() != nil {
+1 -1
View File
@@ -86,7 +86,7 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
blobs := restic.NewBlobSet()
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
entries, err := repo.ListPack(context.TODO(), id, size)
entries, _, err := repo.ListPack(context.TODO(), id, size)
if err != nil {
t.Fatalf("error listing pack %v: %v", id, err)
}
+2 -1
View File
@@ -76,8 +76,9 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
if ids.Has(id) {
blobs, err := repo.ListPack(ctx, id, size)
blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
// ignore errors for broken pack files to be able to salvage as much as possible
return nil
}
packToBlobs[id] = blobs
+8 -12
View File
@@ -783,7 +783,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
worker := func() error {
for fi := range ch {
entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
entries, _, err := r.ListPack(wgCtx, fi.ID, fi.Size)
if err != nil {
debug.Log("unable to list pack file %v", fi.ID.Str())
m.Lock()
@@ -844,13 +844,8 @@ func (r *Repository) prepareCache() error {
packs := r.idx.Packs(restic.NewIDSet())
ids := make(map[string]struct{})
for id := range packs {
ids[id.String()] = struct{}{}
}
// clear old packs
return r.cache.Clear(backend.PackFile, ids)
return r.cache.Clear(restic.PackFile, packs)
}
// SearchKey finds a key with the supplied password, afterwards the config is
@@ -964,11 +959,12 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
})
}
// ListPack returns the list of blobs saved in the pack id.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
// ListPack returns the list of blobs saved in the pack id and the length of
// the pack header.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, uint32, error) {
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
if err != nil {
if r.cache != nil {
// ignore error as there is not much we can do here
@@ -976,9 +972,9 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
}
// retry on error
entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
}
return entries, err
return entries, hdrSize, err
}
// Delete calls backend.Delete() if implemented, and returns an error
+3 -5
View File
@@ -319,8 +319,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
var repoFixture = filepath.Join("testdata", "test-repo.tar.gz")
func TestRepositoryLoadIndex(t *testing.T) {
repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
defer cleanup()
repo, _ := repository.TestFromFixture(t, repoFixture)
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
}
@@ -373,8 +372,7 @@ func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length
}
func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
repodir, cleanup := rtest.Env(t, repoFixture)
defer cleanup()
repodir := rtest.Env(t, repoFixture)
be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2}, t.Logf)
rtest.OK(t, err)
@@ -479,7 +477,7 @@ func TestListPack(t *testing.T) {
return nil
}))
blobs, err := repo.ListPack(context.TODO(), packID, size)
blobs, _, err := repo.ListPack(context.TODO(), packID, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
+3 -3
View File
@@ -105,11 +105,11 @@ func TestRepositoryWithVersion(t testing.TB, version uint) (*Repository, restic.
return repo, &internalRepository{repo}, be
}
func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend, func()) {
repodir, cleanup := test.Env(t, repoFixture)
func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend) {
repodir := test.Env(t, repoFixture)
repo, be := TestOpenLocal(t, repodir)
return repo, be, cleanup
return repo, be
}
// TestOpenLocal opens a local repository.
+2 -2
View File
@@ -24,13 +24,13 @@ func (e *NoIDByPrefixError) Error() string {
// Find loads the list of all files of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned.
func Find(ctx context.Context, repo Lister, t FileType, prefix string) (ID, error) {
func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
match := ID{}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := repo.List(ctx, t, func(id ID, _ int64) error {
err := be.List(ctx, t, func(id ID, _ int64) error {
name := id.String()
if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
if match.IsNull() {
+4
View File
@@ -57,6 +57,10 @@ func (h BlobHandle) String() string {
return fmt.Sprintf("<%s/%s>", h.Type, h.ID.Str())
}
func NewRandomBlobHandle() BlobHandle {
return BlobHandle{ID: NewRandomID(), Type: DataBlob}
}
// BlobType specifies what a blob stored in a pack is.
type BlobType uint8
+5
View File
@@ -26,6 +26,11 @@ const MaxRepoVersion = 2
// is newly created with Init().
const StableRepoVersion = 2
// JSONUnpackedLoader loads unpacked JSON.
type JSONUnpackedLoader interface {
LoadJSONUnpacked(context.Context, FileType, ID, interface{}) error
}
// CreateConfig creates a config file with a randomly selected polynomial and
// ID.
func CreateConfig(version uint) (Config, error) {
+13
View File
@@ -1,9 +1,11 @@
package restic
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
)
// Hash returns the ID for data.
@@ -38,6 +40,17 @@ func (id ID) String() string {
return hex.EncodeToString(id[:])
}
// NewRandomID returns a randomly generated ID. When reading from rand fails,
// the function panics.
func NewRandomID() ID {
id := ID{}
_, err := io.ReadFull(rand.Reader, id[:])
if err != nil {
panic(err)
}
return id
}
const shortStr = 4
// Str returns the shortened string version of id.
+3 -2
View File
@@ -32,8 +32,9 @@ type Repository interface {
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
// ListPack returns the list of blobs saved in the pack id.
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
// ListPack returns the list of blobs saved in the pack id and the length of
// the pack header.
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, hdrSize uint32, err error)
LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error
-17
View File
@@ -1,9 +1,7 @@
package restic
import (
"crypto/rand"
"fmt"
"io"
)
// TestParseID parses s as a ID and panics if that fails.
@@ -20,18 +18,3 @@ func TestParseID(s string) ID {
func TestParseHandle(s string, t BlobType) BlobHandle {
return BlobHandle{ID: TestParseID(s), Type: t}
}
func NewRandomBlobHandle() BlobHandle {
return BlobHandle{ID: NewRandomID(), Type: DataBlob}
}
// NewRandomID returns a randomly generated ID. When reading from rand fails,
// the function panics.
func NewRandomID() ID {
id := ID{}
_, err := io.ReadFull(rand.Reader, id[:])
if err != nil {
panic(err)
}
return id
}
+3
View File
@@ -80,9 +80,12 @@ func extractToFile(buf []byte, filename, target string, printf func(string, ...i
return err
}
if err = newFile.Sync(); err != nil {
_ = newFile.Close()
_ = os.Remove(newFile.Name())
return err
}
if err = newFile.Close(); err != nil {
_ = os.Remove(newFile.Name())
return err
}
+2
View File
@@ -87,6 +87,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
var msg githubError
jerr := json.NewDecoder(res.Body).Decode(&msg)
if jerr == nil {
_ = res.Body.Close()
return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n %v", res.StatusCode, res.Status, msg.Message)
}
}
@@ -137,6 +138,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
}
if res.StatusCode != http.StatusOK {
_ = res.Body.Close()
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
+12 -11
View File
@@ -137,10 +137,18 @@ func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
// Env creates a test environment and extracts the repository fixture.
// Returned is the repo path and a cleanup function.
func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
func Env(t testing.TB, repoFixture string) string {
t.Helper()
tempdir, err := os.MkdirTemp(TestTempDir, "restic-test-env-")
OK(t, err)
var tempdir string
if TestCleanupTempDirs {
tempdir = t.TempDir()
} else {
var err error
tempdir, err = os.MkdirTemp(TestTempDir, "restic-test-env-")
OK(t, err)
t.Logf("leaving temporary directory %v used for test", tempdir)
}
fd, err := os.Open(repoFixture)
if err != nil {
@@ -150,14 +158,7 @@ func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
SetupTarTestFixture(t, tempdir, repoFixture)
return filepath.Join(tempdir, "repo"), func() {
if !TestCleanupTempDirs {
t.Logf("leaving temporary directory %v used for test", tempdir)
return
}
RemoveAll(t, tempdir)
}
return filepath.Join(tempdir, "repo")
}
func isFile(fi os.FileInfo) bool {