Compare commits

..
26 Commits
Author SHA1 Message Date
Michael Eischer abfc9bbdff Merge pull request #21868 from MichaelEischer/fix-zero-sized-blobs
repository: fix zero-sized blobs in v2 repos
2026-06-11 20:43:38 +02:00
Michael Eischer 784812361e Merge pull request #21858 from MichaelEischer/unexport-backends
Unexport several backend types
2026-06-10 22:51:32 +02:00
Michael Eischer c745f810b3 backend/cache: use backend types not restic types (#21860) 2026-06-10 22:45:39 +02:00
Michael Eischer 721411e04c backend/azure/gs/s3: remove unused Path() method 2026-06-10 22:37:07 +02:00
Michael Eischer b07ff6cd34 backends: remove unused SetListMax method
It was never used. Remove the dead code.
2026-06-10 22:37:07 +02:00
Michael Eischer 8b3bbb527c backend/s3: unexport 2026-06-10 22:37:07 +02:00
Michael Eischer 4a2d8e04aa backend/rclone: unexport 2026-06-10 22:37:07 +02:00
Michael Eischer a36488b706 backend/gs: unexport 2026-06-10 22:37:07 +02:00
Michael Eischer cfb506a77a backend/cache: unexport 2026-06-10 22:37:07 +02:00
Michael Eischer a15b1579ff Merge pull request #21846 from restic/repository-cleanups
Cleanup repository package
2026-06-10 22:36:04 +02:00
Michael Eischer ff95080f36 restic: move test helpers and drop unused JSONUnpackedLoader 2026-06-10 22:20:28 +02:00
Michael Eischer 119bb9d9a8 repository: require *Repository for ExaminePack
The function wouldn't work with a different Repository implementation
anyways.
2026-06-10 22:20:28 +02:00
Michael Eischer 8c6ee42d17 debug: move DumpPacks into repository package
Processing pack file internals belongs into the repository package.
2026-06-10 22:20:28 +02:00
Michael Eischer ccddc1914d repository: rename PackError to ErrPackMetadata 2026-06-10 22:20:28 +02:00
Michael Eischer 4bc5eca7ea repository: merge check.go into checker.go 2026-06-10 22:20:28 +02:00
Michael Eischer 17ff3aa5f9 Merge pull request #21841 from MichaelEischer/isolate-repository
prevent imports of repository internals
2026-06-10 22:19:24 +02:00
Michael Eischer 5683180a3b upgrade compress library 2026-06-10 22:03:56 +02:00
Michael Eischer 0fc7444e32 repository: fix zero-sized blobs in v2 repos 2026-06-10 22:03:40 +02:00
Michael Eischer 81571775d5 fix version set by helper after release (#21865) 2026-06-09 22:43:18 +02:00
Alexander Neumann 16b8b8cda0 Set development version for 0.19.0 2026-06-09 18:49:09 +02:00
Michael Eischer 7d25ca9d67 repository: omit unused headerSize from ListPack() 2026-05-31 22:53:00 +02:00
Michael Eischer cc546b71e3 debug: move code requring internal access to repository package 2026-05-31 22:48:38 +02:00
Michael Eischer 3cb49556f5 repair index: replace full index handling integration test with unit test 2026-05-31 22:39:16 +02:00
Michael Eischer f625190393 check: simplify index damage test 2026-05-31 22:19:15 +02:00
Michael Eischer 620f5986f8 list index: use helper in repository package 2026-05-31 22:14:01 +02:00
Michael Eischer 49c7364e79 add golangci-lint rule to prevent imports of repository internals 2026-05-31 18:20:37 +02:00
77 changed files with 1103 additions and 1756 deletions
+7 -1
View File
@@ -32,7 +32,6 @@ linters:
backend-imports:
files:
- "**/internal/backend/**"
- "!**/internal/backend/cache/**"
- "!**/internal/backend/test/**"
- "!**/*_test.go"
deny:
@@ -40,6 +39,13 @@ linters:
desc: "internal/restic should not be imported to keep the architectural layers intact"
- pkg: "github.com/restic/restic/internal/repository"
desc: "internal/repository should not be imported to keep the architectural layers intact"
repository-internals:
files:
- "**"
- "!**/internal/repository/**"
deny:
- pkg: "github.com/restic/restic/internal/repository/"
desc: "packages below internal/repository should not be imported to not depend on repository internals"
importas:
alias:
- pkg: github.com/restic/restic/internal/test
-111
View File
@@ -1,6 +1,5 @@
# Table of Contents
* [Changelog for 0.19.1](#changelog-for-restic-0191-2026-07-05)
* [Changelog for 0.19.0](#changelog-for-restic-0190-2026-06-09)
* [Changelog for 0.18.1](#changelog-for-restic-0181-2025-09-21)
* [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
@@ -42,116 +41,6 @@
* [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
# Changelog for restic 0.19.1 (2026-07-05)
The following sections list the changes in restic 0.19.1 relevant to
restic users. The changes are ordered by importance.
## Summary
* Fix #5234: Prevent mounting over the repository directory
* Fix #5667: Skip inaccessible `backup` source paths
* Fix #5722: Update `mount` latest symlink after snapshot reload
* Fix #21866: Hide `stats` progress bar in JSON mode
* Fix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
* Fix #21876: Show timezone location in `snapshots` output
* Fix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
* Fix #21895: Remove read-only files via the SFTP backend on Windows servers
* Fix #21899: Make `backup` respect excludes for duplicate directory entries
## Details
* Bugfix #5234: Prevent mounting over the repository directory
Using a local repository directory as the `mount` target — or a path that
contains it, or that it contains — caused the FUSE server to read its own
backend files through the new mount, deadlocking the kernel and requiring a long
reboot to recover.
Restic now resolves both paths and refuses any such overlap with a clear error
before mounting.
https://github.com/restic/restic/issues/5234
https://github.com/restic/restic/pull/5348
* Bugfix #5667: Skip inaccessible `backup` source paths
The `backup` command only skipped source paths that did not exist. A path that
could not be accessed for another reason, such as a malformed path on Windows,
was kept and produced an empty snapshot. Restic now skips any such path and
aborts if none remain.
https://github.com/restic/restic/issues/5667
https://github.com/restic/restic/pull/21852
* Bugfix #5722: Update `mount` latest symlink after snapshot reload
When `restic mount` was kept running while new snapshots were created, the new
snapshots appeared in the mountpoint, but the `latest` symlink could still point
to the previously latest snapshot. Restic now invalidates the cached snapshot
directory entries after a snapshot reload so that `latest` points to the newest
snapshot.
https://github.com/restic/restic/issues/5722
https://github.com/restic/restic/pull/21873
* Bugfix #21866: Hide `stats` progress bar in JSON mode
Since restic 0.19.0, the `stats` command shows a progress bar. This progress bar
was unintentionally displayed also when using the `--json` option, mixing
regular text output with JSON. This is now fixed.
https://github.com/restic/restic/issues/21866
https://github.com/restic/restic/pull/21871
* Bugfix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>` to
no longer group snapshots by host and paths by default.
The `snapshots --latest <n>` command now again uses the old behavior of grouping
by host and paths when `--group-by` is not specified. However, when specifying
`--group-by` the output is still grouped as requested, as in restic 0.19.0.
https://github.com/restic/restic/issues/21869
https://github.com/restic/restic/pull/21875
* Bugfix #21876: Show timezone location in `snapshots` output
With restic 0.19.0, the `snapshots` command printed the current timezone when
listing snapshots. However, that timezone label might change during the year,
for example with daylight saving time. Restic now prints a more consistent and
shorter version of the text.
https://github.com/restic/restic/pull/21876
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
* Bugfix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
the repository. If restic was unable to stat the mountpoint, this would result
in a crash. This has now been fixed to correctly return an error instead.
https://github.com/restic/restic/pull/21879
* Bugfix #21895: Remove read-only files via the SFTP backend on Windows servers
Since restic 0.19.0, repository files on the SFTP backend are marked read-only
after save. On Windows SFTP servers, removing them failed with a permission
error. The SFTP backend now clears the read-only flag before removing the file.
https://github.com/restic/restic/issues/21895
https://github.com/restic/restic/pull/21897
* Bugfix #21899: Make `backup` respect excludes for duplicate directory entries
Since restic 0.19.0, backing up a directory with duplicate directory entries
always produced "Warning: at least one source file could not be read", even when
those files were excluded. This has now been fixed.
https://github.com/restic/restic/issues/21899
https://github.com/restic/restic/pull/21900
# Changelog for restic 0.19.0 (2026-06-09)
The following sections list the changes in restic 0.19.0 relevant to
restic users. The changes are ordered by importance.
+1 -1
View File
@@ -1 +1 @@
0.19.1
0.19.0-dev
-8
View File
@@ -1,8 +0,0 @@
Bugfix: Hide `stats` progress bar in JSON mode
Since restic 0.19.0, the `stats` command shows a progress bar. This
progress bar was unintentionally displayed also when using the `--json`
option, mixing regular text output with JSON. This is now fixed.
https://github.com/restic/restic/issues/21866
https://github.com/restic/restic/pull/21871
-12
View File
@@ -1,12 +0,0 @@
Bugfix: Restore old behavior of `snapshots --latest <n>` without `--group-by`
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>`
to no longer group snapshots by host and paths by default.
The `snapshots --latest <n>` command now again uses the old behavior of
grouping by host and paths when `--group-by` is not specified. However, when
specifying `--group-by` the output is still grouped as requested, as in restic
0.19.0.
https://github.com/restic/restic/issues/21869
https://github.com/restic/restic/pull/21875
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Remove read-only files via the SFTP backend on Windows servers
Since restic 0.19.0, repository files on the SFTP backend are marked
read-only after save. On Windows SFTP servers, removing them failed
with a permission error. The SFTP backend now clears the read-only flag
before removing the file.
https://github.com/restic/restic/issues/21895
https://github.com/restic/restic/pull/21897
-8
View File
@@ -1,8 +0,0 @@
Bugfix: Make `backup` respect excludes for duplicate directory entries
Since restic 0.19.0, backing up a directory with duplicate directory
entries always produced "Warning: at least one source file could not be
read", even when those files were excluded. This has now been fixed.
https://github.com/restic/restic/issues/21899
https://github.com/restic/restic/pull/21900
-12
View File
@@ -1,12 +0,0 @@
Bugfix: Prevent mounting over the repository directory
Using a local repository directory as the `mount` target — or a path
that contains it, or that it contains — caused the FUSE server to
read its own backend files through the new mount, deadlocking the
kernel and requiring a long reboot to recover.
Restic now resolves both paths and refuses any such overlap with a
clear error before mounting.
https://github.com/restic/restic/issues/5234
https://github.com/restic/restic/pull/5348
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Skip inaccessible `backup` source paths
The `backup` command only skipped source paths that did not exist. A path that
could not be accessed for another reason, such as a malformed path on Windows,
was kept and produced an empty snapshot. Restic now skips any such path and
aborts if none remain.
https://github.com/restic/restic/issues/5667
https://github.com/restic/restic/pull/21852
-10
View File
@@ -1,10 +0,0 @@
Bugfix: Update `mount` latest symlink after snapshot reload
When `restic mount` was kept running while new snapshots were
created, the new snapshots appeared in the mountpoint, but the `latest`
symlink could still point to the previously latest snapshot. Restic now
invalidates the cached snapshot directory entries after a snapshot reload so
that `latest` points to the newest snapshot.
https://github.com/restic/restic/issues/5722
https://github.com/restic/restic/pull/21873
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Show timezone location in `snapshots` output
With restic 0.19.0, the `snapshots` command printed the current timezone when
listing snapshots. However, that timezone label might change during the year,
for example with daylight saving time. Restic now prints a more consistent and
shorter version of the text.
https://github.com/restic/restic/pull/21876
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Prevent crash in mountpoint validation if mountpoint is inaccessible
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
the repository. If restic was unable to stat the mountpoint, this would result
in a crash. This has now been fixed to correctly return an error instead.
https://github.com/restic/restic/pull/21879
+4 -8
View File
@@ -171,17 +171,13 @@ var ErrInvalidSourceData = errors.New("at least one source file could not be rea
// ErrNoSourceData is used to report that no source data was found
var ErrNoSourceData = errors.Fatal("all source directories/files do not exist")
// filterExisting returns the items that exist and can be accessed. It returns
// ErrNoSourceData if none remain, or ErrInvalidSourceData if some were skipped.
// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string, warnf func(msg string, args ...interface{})) (result []string, err error) {
for _, item := range items {
_, err := fs.Lstat(item)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
warnf("%v does not exist, skipping\n", item)
} else {
warnf("%v cannot be accessed, skipping\n", item)
}
if errors.Is(err, os.ErrNotExist) {
warnf("%v does not exist, skipping\n", item)
continue
}
-23
View File
@@ -10,7 +10,6 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/errors"
rtest "github.com/restic/restic/internal/test"
)
@@ -77,28 +76,6 @@ func TestCollectTargets(t *testing.T) {
rtest.Assert(t, err == ErrInvalidSourceData, "expected error when not all targets exist")
}
func TestFilterExistingUnreadable(t *testing.T) {
dir := rtest.TempDir(t)
existing := filepath.Join(dir, "existing")
rtest.OK(t, os.Mkdir(existing, 0755))
file := filepath.Join(dir, "file")
rtest.OK(t, os.WriteFile(file, []byte("x"), 0600))
// Regression test for #5667. A target whose Lstat fails with an error other
// than ErrNotExist must be skipped (ENOTDIR on unix, NUL byte everywhere).
for _, unreadable := range []string{filepath.Join(file, "child"), "invalid\x00path"} {
result, err := filterExisting([]string{unreadable}, t.Logf)
rtest.Assert(t, errors.Is(err, ErrNoSourceData), "input %q: expected ErrNoSourceData; got %v", unreadable, err)
rtest.Assert(t, len(result) == 0, "input %q: expected no targets; got %v", unreadable, result)
result, err = filterExisting([]string{existing, unreadable}, t.Logf)
rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "input %q: expected ErrInvalidSourceData; got %v", unreadable, err)
rtest.Equals(t, []string{existing}, result)
}
}
func TestReadFilenamesRaw(t *testing.T) {
// These should all be returned exactly as-is.
expected := []string{
+1 -1
View File
@@ -307,7 +307,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
go chkr.Packs(ctx, errChan)
for err := range errChan {
var packErr *repository.PackError
var packErr *repository.ErrPackMetadata
if errors.As(err, &packErr) {
if packErr.Orphaned {
orphanedPacks++
+1 -1
View File
@@ -103,7 +103,7 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
defer unlock()
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(context.TODO(), id, size)
blobs, err := repo.ListPack(context.TODO(), id, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
+10 -365
View File
@@ -4,31 +4,19 @@ package main
import (
"context"
"crypto/aes"
"crypto/cipher"
"encoding/json"
"fmt"
"io"
"os"
"runtime"
"sync"
"time"
"github.com/klauspost/compress/zstd"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
)
func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) {
@@ -128,61 +116,6 @@ func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io
})
}
// Pack is the struct used in printPacks.
type Pack struct {
Name string `json:"name"`
Blobs []Blob `json:"blobs"`
}
// Blob is the struct used in printPacks.
type Blob struct {
Type restic.BlobType `json:"type"`
Length uint `json:"length"`
ID restic.ID `json:"id"`
Offset uint `json:"offset"`
}
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
printer.E("error for pack %v: %v", id.Str(), err)
return nil
}
p := Pack{
Name: id.String(),
Blobs: make([]Blob, len(blobs)),
}
for i, blob := range blobs {
p.Blobs[i] = Blob{
Type: blob.Type,
Length: blob.Length,
ID: blob.ID,
Offset: blob.Offset,
}
}
m.Lock()
defer m.Unlock()
return prettyPrintJSON(wr, p)
})
}
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id)
if err != nil {
return err
}
return idx.Dump(wr)
})
}
func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
@@ -200,11 +133,11 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
switch tpe {
case "indexes":
return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
case "snapshots":
return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
case "packs":
return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
return repository.DumpPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
case "all":
printer.S("snapshots:")
err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
@@ -213,7 +146,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
}
printer.S("indexes:")
err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
if err != nil {
return err
}
@@ -224,225 +157,6 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
}
}
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
if bytewise {
printer.S(" trying to repair blob by finding a broken byte")
} else {
printer.S(" trying to repair blob with single bit flip")
}
ch := make(chan int)
var wg errgroup.Group
done := make(chan struct{})
var fixed []byte
var found bool
workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ {
wg.Go(func() error {
// make a local copy of the buffer
buf := make([]byte, len(input))
copy(buf, input)
testFlip := func(idx int, pattern byte) bool {
// flip bits
buf[idx] ^= pattern
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil {
printer.S("")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext))
close(done)
found = true
fixed = plaintext
return true
}
// flip bits back
buf[idx] ^= pattern
return false
}
for i := range ch {
if bytewise {
for j := 0; j < 255; j++ {
if testFlip(i, byte(j)) {
return nil
}
}
} else {
for j := 0; j < 7; j++ {
// flip each bit once
if testFlip(i, (1 << uint(j))) {
return nil
}
}
}
}
return nil
})
}
wg.Go(func() error {
defer close(ch)
start := time.Now()
info := time.Now()
for i := range input {
select {
case ch <- i:
case <-done:
printer.S(" done after %v", time.Since(start))
return nil
}
if time.Since(info) > time.Second {
secs := time.Since(start).Seconds()
gps := float64(i) / secs
remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now()
}
}
return nil
})
err := wg.Wait()
if err != nil {
panic("all go routines can only return nil")
}
if !found {
printer.S("\n blob could not be repaired")
}
return fixed
}
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
// strip signature at the end
l := len(buf)
nonce, ct := buf[:16], buf[16:l-16]
out := make([]byte, len(ct))
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCTR(c, nonce)
e.XORKeyStream(out, ct)
return out
}
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
// allow processing broken pack files
if pack == nil {
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(pack) {
printer.E("skipping truncated blob")
continue
}
buf := pack[blob.Offset : blob.Offset+blob.Length]
key := repo.Key()
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
outputPrefix := ""
filePrefix := ""
if err != nil {
printer.E("error decrypting blob: %v", err)
if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
if err != nil {
return err
}
continue
}
}
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
printer.S(" failed to decompress blob %v", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
}
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
if err != nil {
return err
}
}
if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
printer.S(" uploaded %v %v", blob.Type, id)
}
}
return nil
})
return err
}
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename)
if err != nil {
return err
}
_, err = f.Write(plain)
if err != nil {
_ = f.Close()
return err
}
err = f.Close()
if err != nil {
return err
}
printer.S("decrypt of blob %v stored at %v", id, filename)
return nil
}
func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
@@ -478,8 +192,14 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
return err
}
examineOpts := repository.ExaminePackOptions{
TryRepair: opts.TryRepair,
RepairByte: opts.RepairByte,
ExtractPack: opts.ExtractPack,
ReuploadBlobs: opts.ReuploadBlobs,
}
for _, id := range ids {
err := examinePack(ctx, opts, repo, id, printer)
err := repository.ExaminePack(ctx, repo, id, examineOpts, printer)
if err != nil {
printer.E("error: %v", err)
}
@@ -489,78 +209,3 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
}
return nil
}
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
printer.S("examine %v", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
printer.S(" file size is %v", len(buf))
gotID := restic.Hash(buf)
if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID)
} else {
printer.S(" hash for file content matches")
}
printer.S(" ========================================")
printer.S(" looking for info in the indexes")
blobsLoaded := false
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
continue
}
checkPackSize(blobs, len(buf), printer)
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
if err != nil {
printer.E("error: %v", err)
} else {
blobsLoaded = true
}
}
printer.S(" ========================================")
printer.S(" inspect the pack itself")
blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
checkPackSize(blobs, len(buf), printer)
if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer)
}
return nil
}
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
// track current size and offset
var size, offset uint64
blobs.Sort()
for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
}
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
}
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
} else {
printer.S(" file sizes match")
}
}
+1 -1
View File
@@ -472,7 +472,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
delete(packIDs, idStr)
}
debug.Log("Found pack %s", idStr)
blobs, _, err := f.repo.ListPack(ctx, id, size)
blobs, err := f.repo.ListPack(ctx, id, size)
if err != nil {
return err
}
+1 -1
View File
@@ -261,7 +261,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
}
var key data.SnapshotGroupKey
if err := json.Unmarshal([]byte(k), &key); err != nil {
if json.Unmarshal([]byte(k), &key) != nil {
return err
}
+7 -12
View File
@@ -6,7 +6,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
@@ -69,18 +69,13 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T
case "locks":
t = restic.LockFile
case "blobs":
return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
if err != nil {
return err
for entry := range repository.AllIndexBlobs(ctx, repo, repo) {
if entry.Error != nil {
return entry.Error
}
for blobs := range idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
}
printer.S("%v %v", blobs.Type, blobs.ID)
}
return nil
})
printer.S("%v %v", entry.Handle.Type, entry.Handle.ID)
}
return nil
default:
return errors.Fatal("invalid type")
}
+16 -93
View File
@@ -6,7 +6,6 @@ import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
@@ -14,8 +13,6 @@ import (
"github.com/spf13/pflag"
"golang.org/x/sys/unix"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -134,8 +131,22 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
}
mountpoint := args[0]
if err := validateMountpoint(mountpoint, gopts); err != nil {
return err
// Check the existence of the mount point at the earliest stage to
// prevent unnecessary computations while opening the repository.
stat, err := os.Stat(mountpoint)
if errors.Is(err, os.ErrNotExist) {
printer.P("Mountpoint %s doesn't exist", mountpoint)
return errors.Fatal("invalid mountpoint")
} else if !stat.IsDir() {
printer.P("Mountpoint %s is not a directory", mountpoint)
return errors.Fatal("invalid mountpoint")
}
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
if err != nil {
printer.P("Mountpoint %s is not writeable or not executable", mountpoint)
return errors.Fatal("inaccessible mountpoint")
}
debug.Log("start mount")
@@ -219,91 +230,3 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
return err
}
func validateMountpoint(mountpoint string, gopts global.Options) error {
// Check the existence of the mount point at the earliest stage to
// prevent unnecessary computations while opening the repository.
stat, err := os.Stat(mountpoint)
if errors.Is(err, os.ErrNotExist) {
return errors.Fatal(fmt.Sprintf("mountpoint %s does not exist", mountpoint))
} else if err != nil {
return errors.Fatal(fmt.Sprintf("mountpoint %s is inaccessible: %v", mountpoint, err))
} else if !stat.IsDir() {
return errors.Fatal(fmt.Sprintf("mountpoint %s is not a directory", mountpoint))
}
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
if err != nil {
return errors.Fatal(fmt.Sprintf("mountpoint %s is not writeable or not executable", mountpoint))
}
// Refuse to mount onto (or under, or over) the local repository directory.
// Doing so makes the FUSE server read its own backend files through the
// mount it just created, deadlocking the kernel (GH #5234).
loc, err := location.Parse(gopts.Backends, gopts.Repo)
if err != nil {
return err
}
if loc.Scheme == "local" {
if err := checkMountpointOverlap(loc.Config.(*local.Config).Path, mountpoint); err != nil {
return err
}
}
return nil
}
// checkMountpointOverlap returns an error.Fatal if the local repository at
// repoPath and the mountpoint overlap: equal paths, mountpoint nested inside
// the repo, or the repo nested inside the mountpoint. Any overlap deadlocks
// the FUSE server (GH #5234).
func checkMountpointOverlap(repoPath, mountpoint string) error {
rp, err := resolvePath(repoPath)
if err != nil {
return err
}
mp, err := resolvePath(mountpoint)
if err != nil {
return err
}
const tail = "; refusing to mount to avoid deadlocking the FUSE server"
switch {
case rp == mp:
return errors.Fatal(fmt.Sprintf("mountpoint %s is the local repository directory%s", mp, tail))
case isInside(rp, mp):
return errors.Fatal(fmt.Sprintf("mountpoint %s is inside the local repository directory %s%s", mp, rp, tail))
case isInside(mp, rp):
return errors.Fatal(fmt.Sprintf("local repository directory %s is inside the mountpoint %s%s", rp, mp, tail))
}
return nil
}
// resolvePath returns p as an absolute, symlink-resolved path. If EvalSymlinks
// fails (e.g. the path does not fully exist), it falls back to the absolute
// form: overlap detection is best-effort and we'd rather refuse a clear
// overlap than abort on an unrelated stat error.
func resolvePath(p string) (string, error) {
abs, err := filepath.Abs(p)
if err != nil {
return "", err
}
resolved, err := filepath.EvalSymlinks(abs)
if err != nil {
return abs, nil
}
return resolved, nil
}
// isInside reports whether child is strictly nested inside parent. Both paths
// must already be cleaned and absolute. Equal paths return false; the caller
// handles equality separately so it can produce a distinct error message.
func isInside(parent, child string) bool {
rel, err := filepath.Rel(parent, child)
if err != nil {
return false
}
if rel == "." || rel == ".." {
return false
}
return !strings.HasPrefix(rel, ".."+string(filepath.Separator))
}
-57
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -213,62 +212,6 @@ func TestMount(t *testing.T) {
checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 4)
}
func TestCheckMountpointOverlap(t *testing.T) {
tempdir := t.TempDir()
repo := filepath.Join(tempdir, "repo")
repoSub := filepath.Join(repo, "sub")
sibling := filepath.Join(tempdir, "mnt")
for _, d := range []string{repo, repoSub, sibling} {
rtest.OK(t, os.MkdirAll(d, 0700))
}
cases := []struct {
name string
repo string
mount string
wantSub string // substring of expected error; empty means expect nil
}{
{"equal", repo, repo, "is the local repository directory"},
{"mount inside repo", repo, repoSub, "is inside the local repository directory"},
{"repo inside mount", repoSub, repo, "is inside the mountpoint"},
{"disjoint", repo, sibling, ""},
{"prefix-not-subpath", repo, repo + "-other", ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rtest.OK(t, os.MkdirAll(tc.mount, 0700))
err := checkMountpointOverlap(tc.repo, tc.mount)
if tc.wantSub == "" {
rtest.OK(t, err)
return
}
if err == nil {
t.Fatalf("expected error containing %q, got nil", tc.wantSub)
}
if !strings.Contains(err.Error(), tc.wantSub) {
t.Fatalf("error %q does not contain %q", err.Error(), tc.wantSub)
}
})
}
}
func TestCheckMountpointOverlapSymlink(t *testing.T) {
tempdir := t.TempDir()
repo := filepath.Join(tempdir, "repo")
rtest.OK(t, os.MkdirAll(repo, 0700))
link := filepath.Join(tempdir, "link-to-repo")
rtest.OK(t, os.Symlink(repo, link))
err := checkMountpointOverlap(repo, link)
if err == nil {
t.Fatal("expected overlap error when mountpoint is a symlink to repo, got nil")
}
if !strings.Contains(err.Error(), "is the local repository directory") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestMountSameTimestamps(t *testing.T) {
if !rtest.RunFuseTest {
t.Skip("Skipping fuse tests")
@@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -61,15 +60,6 @@ func TestRebuildIndex(t *testing.T) {
testRebuildIndex(t, nil)
}
func TestRebuildIndexAlwaysFull(t *testing.T) {
indexFull := index.Full
defer func() {
index.Full = indexFull
}()
index.Full = func(*index.Index) bool { return true }
testRebuildIndex(t, nil)
}
// indexErrorBackend modifies the first index after reading.
type indexErrorBackend struct {
backend.Backend
+12 -59
View File
@@ -38,9 +38,6 @@ Exit status is 12 if the password is incorrect.
`,
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
PreRunE: func(_ *cobra.Command, _ []string) error {
return opts.Finalize()
},
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runSnapshots(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
@@ -55,7 +52,7 @@ Exit status is 12 if the password is incorrect.
type SnapshotOptions struct {
data.SnapshotFilter
Compact bool
last bool // Deprecated in favour of Latest.
Last bool // This option should be removed in favour of Latest.
Latest int
GroupBy data.SnapshotGroupByOptions
}
@@ -63,7 +60,7 @@ type SnapshotOptions struct {
func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format")
f.BoolVar(&opts.last, "last", false, "only show the last snapshot for each host and path")
f.BoolVar(&opts.Last, "last", false, "only show the last snapshot for each host and path")
err := f.MarkDeprecated("last", "use --latest 1")
if err != nil {
// MarkDeprecated only returns an error when the flag is not found
@@ -73,13 +70,6 @@ func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma")
}
func (opts *SnapshotOptions) Finalize() error {
if opts.last && opts.Latest == 0 {
opts.Latest = 1
}
return nil
}
func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
@@ -105,12 +95,12 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
return ctx.Err()
}
if opts.Latest > 0 {
if grouped {
list = filterLatestSnapshotsInGroup(list, opts.Latest)
} else {
list = filterLatestSnapshots(list, opts.Latest)
}
if opts.Last {
// This branch should be removed in the same time
// that --last.
list = filterLatestSnapshotsInGroup(list, 1)
} else if opts.Latest > 0 {
list = filterLatestSnapshotsInGroup(list, opts.Latest)
}
sort.Sort(sort.Reverse(list))
snapshotGroups[k] = list
@@ -144,43 +134,6 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
return nil
}
// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
Hostname string
JoinedPaths string
}
// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *data.Snapshot) filterLastSnapshotsKey {
// Shallow slice copy
var paths = make([]string, len(sn.Paths))
copy(paths, sn.Paths)
sort.Strings(paths)
return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}
// filterLatestSnapshots filters a list of snapshots to only return
// the limit last entries for each hostname and path. If the snapshot
// contains multiple paths, they will be joined and treated as one
// item.
func filterLatestSnapshots(list data.Snapshots, limit int) data.Snapshots {
// Sort the snapshots so that the newer ones are listed first
sort.SliceStable(list, func(i, j int) bool {
return list[i].Time.After(list[j].Time)
})
var results data.Snapshots
seen := make(map[filterLastSnapshotsKey]int)
for _, sn := range list {
key := newFilterLastSnapshotsKey(sn)
if seen[key] < limit {
seen[key]++
results = append(results, sn)
}
}
return results
}
// filterLatestSnapshotsInGroup filters a list of snapshots to only return
// the `limit` last entries. It is assumed that the snapshot list only contains
// one group of snapshots.
@@ -292,11 +245,11 @@ func PrintSnapshots(stdout io.Writer, list data.Snapshots, reasons []data.KeepRe
// Each snapshot can be registered in different timezones,
// but we display them all in local timezone on this output.
footer := fmt.Sprintf("%d snapshots", len(list))
zoneName := time.Now().Local().Location().String()
if zoneName == "Local" {
zoneName = "local time"
zoneName, _ := time.Now().Local().Zone()
if zoneName != "" {
footer = fmt.Sprintf("Timestamps shown in %s timezone\n%s", zoneName, footer)
}
tab.AddFooter(fmt.Sprintf("Timestamps shown in %s\n%s", zoneName, footer))
tab.AddFooter(footer)
if multiline {
// print an additional blank line between snapshots
+5 -33
View File
@@ -35,7 +35,10 @@ func testRunSnapshots(t testing.TB, gopts global.Options) (newest *Snapshot, sna
return
}
func snapshotsGroupTestData(t *testing.T, env *testEnvironment, keepPath bool) string {
func TestSnapshotsGroupByAndLatest(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
// two backups on the same host but with different paths
opts := BackupOptions{Host: "testhost", TimeStamp: time.Now().Format(time.DateTime)}
@@ -43,22 +46,9 @@ func snapshotsGroupTestData(t *testing.T, env *testEnvironment, keepPath bool) s
// Use later timestamp for second backup
opts.TimeStamp = time.Now().Add(time.Second).Format(time.DateTime)
snapshotsIDs := loadSnapshotMap(t, env.gopts)
targets := []string{"testdata/0"}
if keepPath {
targets = []string{"testdata"}
}
testRunBackup(t, filepath.Dir(env.testdata), targets, opts, env.gopts)
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0"}, opts, env.gopts)
_, secondSnapshotID := lastSnapshot(snapshotsIDs, loadSnapshotMap(t, env.gopts))
return secondSnapshotID
}
func TestSnapshotsGroupByAndLatest(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
secondSnapshotID := snapshotsGroupTestData(t, env, false)
buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
gopts.JSON = true
// only group by host but not path
@@ -75,21 +65,3 @@ func TestSnapshotsGroupByAndLatest(t *testing.T) {
rtest.Assert(t, len(snapshots[0].Snapshots) == 1, "expected only one latest snapshot, got %d", len(snapshots[0].Snapshots))
rtest.Equals(t, snapshots[0].Snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
}
func TestSnapshotsLatest(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
secondSnapshotID := snapshotsGroupTestData(t, env, true)
buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
gopts.JSON = true
opts := SnapshotOptions{Latest: 1}
return runSnapshots(ctx, opts, gopts, []string{}, gopts.Term)
})
rtest.OK(t, err)
snapshots := []Snapshot{}
rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
rtest.Assert(t, len(snapshots) == 1, "expected only one snapshot, got %d", len(snapshots))
rtest.Equals(t, snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
}
+2 -7
View File
@@ -141,7 +141,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
snapshots = append(snapshots, sn)
}
statsProgress := newStatsProgress(term, !gopts.JSON, uint64(len(snapshots)))
statsProgress := newStatsProgress(term, uint64(len(snapshots)))
updater := progress.NewUpdater(ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()), func(runtime time.Duration, final bool) {
statsProgress.printProgress(runtime, final)
@@ -375,7 +375,6 @@ type statsProgress struct {
term ui.Terminal
m sync.Mutex
snapshotCount uint64
show bool
processedSnapshotCount uint64
processedFileCount uint64
@@ -383,18 +382,14 @@ type statsProgress struct {
processedSize uint64
}
func newStatsProgress(term ui.Terminal, show bool, snapshotCount uint64) *statsProgress {
func newStatsProgress(term ui.Terminal, snapshotCount uint64) *statsProgress {
return &statsProgress{
term: term,
show: show,
snapshotCount: snapshotCount,
}
}
func (s *statsProgress) printProgress(runtime time.Duration, final bool) {
if !s.show {
return
}
s.m.Lock()
progressBase := s.processedSnapshotCount
+1 -10
View File
@@ -66,7 +66,7 @@ func TestSizeHistogramString(t *testing.T) {
func TestStatsProgress(t *testing.T) {
term := &ui.MockTerminal{}
progress := newStatsProgress(term, true, 2)
progress := newStatsProgress(term, 2)
progress.printProgress(0*time.Second, false)
rtest.Equals(t, []string{"[0:00] 0.00% 0 / 2 snapshots, 0 B"}, term.Output)
@@ -88,12 +88,3 @@ func TestStatsProgress(t *testing.T) {
progress.printProgress(20*time.Second, true)
rtest.Equals(t, []string{"[0:20] 100.00% 2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output)
}
func TestStatsProgressJSON(t *testing.T) {
term := &ui.MockTerminal{}
progress := newStatsProgress(term, false, 2)
progress.printProgress(0*time.Second, false)
// JSON output is not available yet, so just make sure to not break normal json output
rtest.Equals(t, nil, term.Output)
}
-6
View File
@@ -208,12 +208,6 @@ command needs to be in the ``PATH``. On macOS, you need `FUSE-T
<https://www.fuse-t.org/>`__ or `FUSE for macOS <https://osxfuse.github.io/>`__.
On FreeBSD, you may need to install FUSE and load the kernel module (``kldload fuse``).
.. note:: The mountpoint must not overlap the local repository directory.
Using the repository directory itself, a subdirectory of it, or a parent
of it as the mountpoint causes the FUSE server to read its own backend
files through the new mount and deadlock the kernel. ``restic mount``
detects this and refuses such mountpoints.
Restic supports storage and preservation of hard links. However, since
hard links exist in the scope of a filesystem by definition, restoring
hard links from a FUSE mount should be done by a program that preserves
-9
View File
@@ -320,8 +320,6 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
finder := data.NewTreeFinder(previous)
defer finder.Close()
var lastExcluded string
for _, name := range names {
// test if context has been cancelled
if ctx.Err() != nil {
@@ -329,12 +327,6 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
return futureNode{}, ctx.Err()
}
if name == lastExcluded {
// Skip duplicate directory entry if it was already excluded.
// This avoids printing errors about duplicate directory entries even though the entry in question is ignored.
continue
}
pathname := arch.FS.Join(dir, name)
oldNode, err := finder.Find(name)
err = arch.error(pathname, err)
@@ -356,7 +348,6 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
}
if excluded {
lastExcluded = name
continue
}
-84
View File
@@ -894,90 +894,6 @@ func TestArchiverSaveDir(t *testing.T) {
}
}
type duplicateReaddirFS struct {
fs.FS
dir string
names []string
}
func (d *duplicateReaddirFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
f, err := d.FS.OpenFile(name, flag, metadataOnly)
if err != nil {
return nil, err
}
if name == d.dir {
return &duplicateReaddirFile{File: f, names: d.names}, nil
}
return f, nil
}
type duplicateReaddirFile struct {
fs.File
names []string
}
func (f *duplicateReaddirFile) Readdirnames(int) ([]string, error) {
return append([]string(nil), f.names...), nil
}
func TestArchiverSaveDirDuplicateExcludedEntry(t *testing.T) {
const targetNodeName = "targetdir"
src := TestDir{
"excluded": TestFile{Content: "skip me"},
"keep": TestFile{Content: "keep me"},
}
tempdir, repo := prepareTempdirRepoSrc(t, src)
testFS := fs.Track{FS: &duplicateReaddirFS{
FS: &fs.Local{},
dir: ".",
names: []string{"excluded", "excluded", "keep"},
}}
arch := New(repo, testFS, Options{})
arch.summary = &Summary{}
arch.Select = func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
return filepath.Base(item) != "excluded"
}
arch.Error = func(item string, err error) error {
t.Errorf("unexpected archiver error for %v: %v", item, err)
return err
}
back := rtest.Chdir(t, tempdir)
defer back()
// duplicate node check in tree finder is only done if the previous tree is not nil
previousTree, err := data.NewTreeNodeIterator(strings.NewReader(`{"nodes":[]}`))
rtest.OK(t, err)
var treeID restic.ID
err = repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
arch.runWorkers(ctx, wg, uploader)
meta, err := testFS.OpenFile(".", fs.O_NOFOLLOW, true)
rtest.OK(t, err)
ft, err := arch.saveDir(ctx, "/", ".", meta, previousTree, nil)
rtest.OK(t, err)
rtest.OK(t, meta.Close())
fnr := ft.take(ctx)
node := fnr.node
node.Name = targetNodeName
treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{node})
arch.stopWorkers()
return wg.Wait()
})
rtest.OK(t, err)
TestEnsureTree(context.TODO(), t, "/", repo, treeID, TestDir{
"targetdir": TestDir{
"keep": TestFile{Content: "keep me"},
},
})
}
func TestArchiverSaveDirIncremental(t *testing.T) {
tempdir := rtest.TempDir(t)
+9 -22
View File
@@ -31,11 +31,9 @@ import (
// Backend stores data on an azure endpoint.
type Backend struct {
cfg Config
container *azContainer.Client
connections uint
prefix string
listMaxItems int
cfg Config
container *azContainer.Client
connections uint
layout.Layout
accessTier blob.AccessTier
@@ -145,12 +143,11 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
be := &Backend{
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
accessTier: accessTier,
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
accessTier: accessTier,
}
return be, nil
@@ -195,11 +192,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
return bloberror.HasCode(err, bloberror.BlobNotFound)
@@ -231,11 +223,6 @@ func (be *Backend) Hasher() hash.Hash {
return md5.New()
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.prefix
}
// useAccessTier determines whether to apply the configured access tier to a given file.
// For archive access tier, only data files are stored using that class; metadata
// must remain instantly accessible.
@@ -419,7 +406,7 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
prefix += "/"
}
maxI := int32(be.listMaxItems)
maxI := int32(defaultListMaxItems)
opts := &azContainer.ListBlobsFlatOptions{
MaxResults: &maxI,
+13 -22
View File
@@ -23,10 +23,9 @@ import (
// b2Backend is a backend which stores its data on Backblaze B2.
type b2Backend struct {
client *b2.Client
bucket *b2.Bucket
cfg Config
listMaxItems int
client *b2.Client
bucket *b2.Bucket
cfg Config
layout.Layout
canDelete bool
@@ -107,12 +106,11 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string,
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
canDelete: true,
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
canDelete: true,
}
return be, nil
@@ -140,20 +138,14 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
}
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *b2Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
func (be *b2Backend) Properties() backend.Properties {
return backend.Properties{
Connections: be.cfg.Connections,
@@ -246,7 +238,6 @@ func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewi
// sanity check
if n != rd.Length() {
_ = w.Close()
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
}
return errors.Wrap(w.Close(), "Close")
@@ -305,7 +296,7 @@ func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backe
defer cancel()
prefix, _ := be.Basedir(t)
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems))
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(defaultListMaxItems))
for iter.Next() {
obj := iter.Object()
+18 -26
View File
@@ -7,11 +7,10 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend wraps a restic.Backend and adds a cache.
type Backend struct {
// cacheBackend wraps a restic.cacheBackend and adds a cache.
type cacheBackend struct {
backend.Backend
*Cache
@@ -24,10 +23,10 @@ type Backend struct {
}
// ensure Backend implements backend.Backend
var _ backend.Backend = &Backend{}
var _ backend.Backend = &cacheBackend{}
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *Backend {
return &Backend{
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *cacheBackend {
return &cacheBackend{
Backend: be,
Cache: c,
inProgress: make(map[backend.Handle]chan struct{}),
@@ -36,7 +35,7 @@ func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface
}
// Remove deletes a file from the backend and the cache if it has been cached.
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
func (b *cacheBackend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("cache Remove(%v)", h)
err := b.Backend.Remove(ctx, h)
if err != nil {
@@ -58,7 +57,7 @@ func autoCacheTypes(h backend.Handle) bool {
}
// Save stores a new file in the backend and the cache.
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if !autoCacheTypes(h) {
return b.Backend.Save(ctx, h, rd)
}
@@ -92,7 +91,7 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR
return nil
}
func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
finish := make(chan struct{})
b.inProgressMutex.Lock()
@@ -136,7 +135,7 @@ func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
}
// loadFromCache will try to load the file from the cache.
func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
rd, inCache, err := b.Cache.load(h, length, offset)
if err != nil {
return inCache, err
@@ -151,7 +150,7 @@ func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, cons
}
// Load loads a file from the cache or the backend.
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
b.inProgressMutex.Lock()
waitForFinish, inProgress := b.inProgress[h]
b.inProgressMutex.Unlock()
@@ -198,7 +197,7 @@ func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset
// Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache.
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("cache Stat(%v)", h)
fi, err := b.Backend.Stat(ctx, h)
@@ -211,31 +210,24 @@ func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo,
}
// IsNotExist returns true if the error is caused by a non-existing file.
func (b *Backend) IsNotExist(err error) bool {
func (b *cacheBackend) IsNotExist(err error) bool {
return b.Backend.IsNotExist(err)
}
func (b *Backend) Unwrap() backend.Backend {
func (b *cacheBackend) Unwrap() backend.Backend {
return b.Backend
}
func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
if !b.Cache.canBeCached(t) {
return b.Backend.List(ctx, t, fn)
}
// will contain the IDs of the files that are in the repository
ids := restic.NewIDSet()
ids := make(map[string]struct{})
// wrap the original function to also add the file to the ids set
wrapFn := func(f backend.FileInfo) error {
id, err := restic.ParseID(f.Name)
if err != nil {
// ignore files with invalid name
return nil
}
ids.Insert(id)
ids[f.Name] = struct{}{}
// execute the original function
return fn(f)
@@ -260,11 +252,11 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen
}
// Warmup delegates to wrapped backend.
func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
func (b *cacheBackend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
return b.Backend.Warmup(ctx, h)
}
// WarmupWait delegates to wrapped backend.
func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error {
func (b *cacheBackend) WarmupWait(ctx context.Context, h []backend.Handle) error {
return b.Backend.WarmupWait(ctx, h)
}
+4 -5
View File
@@ -12,7 +12,6 @@ import (
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Cache manages a local cache.
@@ -43,10 +42,10 @@ func readVersion(dir string) (v uint, err error) {
const cacheVersion = 1
var cacheLayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshots",
restic.IndexFile: "index",
var cacheLayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshots",
backend.IndexFile: "index",
}
const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"
+7 -12
View File
@@ -12,7 +12,6 @@ import (
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
func (c *Cache) filename(h backend.Handle) string {
@@ -171,7 +170,7 @@ func (c *Cache) remove(h backend.Handle) (bool, error) {
// Clear removes all files of type t from the cache that are not contained in
// the set valid.
func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
debug.Log("Clearing cache for %v: %v valid files", t, len(valid))
if !c.canBeCached(t) {
return nil
@@ -183,12 +182,12 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
}
for id := range list {
if valid.Has(id) {
if _, ok := valid[id]; ok {
continue
}
// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id})); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
@@ -201,12 +200,12 @@ func isFile(fi os.FileInfo) bool {
}
// list returns a list of all files of type T in the cache.
func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
if !c.canBeCached(t) {
return nil, errors.New("cannot be cached")
}
list := restic.NewIDSet()
list := make(map[string]struct{})
dir := filepath.Join(c.path, cacheLayoutPaths[t])
err := filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
if err != nil {
@@ -221,12 +220,8 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
return nil
}
id, err := restic.ParseID(filepath.Base(name))
if err != nil {
return nil
}
list.Insert(id)
id := filepath.Base(name)
list[id] = struct{}{}
return nil
})
+19 -18
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"io"
"maps"
"math/rand"
"os"
"runtime"
@@ -18,8 +19,8 @@ import (
"golang.org/x/sync/errgroup"
)
func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet {
ids := restic.NewIDSet()
func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} {
ids := make(map[string]struct{})
for i := 0; i < random.Intn(15)+10; i++ {
buf := rtest.Random(random.Int(), 1<<19)
id := restic.Hash(buf)
@@ -33,13 +34,13 @@ func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType,
if err != nil {
t.Fatal(err)
}
ids.Insert(id)
ids[id.String()] = struct{}{}
}
return ids
}
// randomID returns a random ID from s.
func randomID(s restic.IDSet) restic.ID {
func randomID(s map[string]struct{}) string {
for id := range s {
return id
}
@@ -69,7 +70,7 @@ func load(t testing.TB, c *Cache, h backend.Handle) []byte {
return buf
}
func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{} {
list, err := c.list(tpe)
if err != nil {
t.Errorf("listing failed: %v", err)
@@ -78,7 +79,7 @@ func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
return list
}
func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
func clearFiles(t testing.TB, c *Cache, tpe backend.FileType, valid map[string]struct{}) {
if err := c.Clear(tpe, valid); err != nil {
t.Error(err)
}
@@ -102,34 +103,34 @@ func TestFiles(t *testing.T) {
ids := generateRandomFiles(t, random, tpe, c)
id := randomID(ids)
h := backend.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id}
id2 := restic.Hash(load(t, c, h))
if !id.Equal(id2) {
t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
if id != id2.String() {
t.Errorf("wrong data returned, want %v, got %v", id, id2.String())
}
if !c.Has(h) {
t.Errorf("cache thinks index %v isn't present", id.Str())
t.Errorf("cache thinks index %v isn't present", id)
}
list := listFiles(t, c, tpe)
if !ids.Equals(list) {
if !maps.Equal(ids, list) {
t.Errorf("wrong list of index IDs returned, want:\n %v\ngot:\n %v", ids, list)
}
clearFiles(t, c, tpe, restic.NewIDSet(id))
clearFiles(t, c, tpe, map[string]struct{}{id: {}})
list2 := listFiles(t, c, tpe)
ids.Delete(id)
want := restic.NewIDSet(id)
if !list2.Equals(want) {
delete(ids, id)
want := map[string]struct{}{id: {}}
if !maps.Equal(list2, want) {
t.Errorf("ClearIndexes removed indexes, want:\n %v\ngot:\n %v", list2, want)
}
clearFiles(t, c, tpe, restic.NewIDSet())
want = restic.NewIDSet()
clearFiles(t, c, tpe, map[string]struct{}{})
want = map[string]struct{}{}
list3 := listFiles(t, c, tpe)
if !list3.Equals(want) {
if !maps.Equal(list3, want) {
t.Errorf("ClearIndexes returned a wrong list, want:\n %v\ngot:\n %v", want, list3)
}
})
+3 -2
View File
@@ -3,16 +3,17 @@ package cache
import (
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
const testCacheID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
// TestNewCache returns a cache in a temporary directory which is removed when
// cleanup is called.
func TestNewCache(t testing.TB) *Cache {
dir := test.TempDir(t)
t.Logf("created new cache at %v", dir)
cache, err := New(restic.NewRandomID().String(), dir)
cache, err := New(testCacheID, dir)
if err != nil {
t.Fatal(err)
}
+33 -49
View File
@@ -27,27 +27,25 @@ import (
"google.golang.org/api/option"
)
// Backend stores data in a GCS bucket.
// gs stores data in a GCS bucket.
//
// The service account used to access the bucket must have these permissions:
// - storage.objects.create
// - storage.objects.delete
// - storage.objects.get
// - storage.objects.list
type Backend struct {
gcsClient *storage.Client
projectID string
connections uint
bucketName string
region string
bucket *storage.BucketHandle
prefix string
listMaxItems int
type gs struct {
gcsClient *storage.Client
projectID string
connections uint
bucketName string
region string
bucket *storage.BucketHandle
layout.Layout
}
// Ensure that *Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
var _ backend.Backend = &gs{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@@ -86,7 +84,7 @@ func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
return gcsClient, nil
}
func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
_, err := bucket.Attrs(ctx)
if err == storage.ErrBucketNotExist {
return false, nil
@@ -94,9 +92,7 @@ func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandl
return err == nil, err
}
const defaultListMaxItems = 1000
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
func open(cfg Config, rt http.RoundTripper) (*gs, error) {
debug.Log("open, config %#v", cfg)
gcsClient, err := getStorageClient(rt)
@@ -104,16 +100,14 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
return nil, errors.Wrap(err, "getStorageClient")
}
be := &Backend{
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
prefix: cfg.Prefix,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
be := &gs{
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
}
return be, nil
@@ -161,17 +155,12 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
func (be *gs) IsNotExist(err error) bool {
return errors.Is(err, storage.ErrObjectNotExist)
}
func (be *Backend) IsPermanentError(err error) bool {
func (be *gs) IsPermanentError(err error) bool {
if be.IsNotExist(err) {
return true
}
@@ -186,7 +175,7 @@ func (be *Backend) IsPermanentError(err error) bool {
return false
}
func (be *Backend) Properties() backend.Properties {
func (be *gs) Properties() backend.Properties {
return backend.Properties{
Connections: be.connections,
HasAtomicReplace: true,
@@ -194,17 +183,12 @@ func (be *Backend) Properties() backend.Properties {
}
// Hasher may return a hash function for calculating a content hash for the backend
func (be *Backend) Hasher() hash.Hash {
func (be *gs) Hasher() hash.Hash {
return md5.New()
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.prefix
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
// Set chunk size to zero to disable resumable uploads.
@@ -254,14 +238,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *gs) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length == 0 {
// negative length indicates read till end to GCS lib
length = -1
@@ -283,7 +267,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
attr, err := be.bucket.Object(objName).Attrs(ctx)
@@ -296,7 +280,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.bucket.Object(objName).Delete(ctx)
@@ -310,7 +294,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@@ -355,15 +339,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
}
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
func (be *gs) Delete(ctx context.Context) error {
return util.DefaultDelete(ctx, be)
}
// Close does nothing.
func (be *Backend) Close() error { return nil }
func (be *gs) Close() error { return nil }
// Warmup not implemented
func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
func (be *gs) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
return []backend.Handle{}, nil
}
func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
func (be *gs) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
+8 -8
View File
@@ -28,8 +28,8 @@ import (
"golang.org/x/net/http2"
)
// Backend is used to access data stored somewhere via rclone.
type Backend struct {
// rclone is used to access data stored somewhere via rclone.
type rclone struct {
*rest.Backend
tr *http2.Transport
cmd *exec.Cmd
@@ -141,7 +141,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn {
}
// New initializes a Backend and starts the process.
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*rclone, error) {
var (
args []string
err error
@@ -196,7 +196,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
}
cmd := stdioConn.cmd
be := &Backend{
be := &rclone{
tr: tr,
cmd: cmd,
waitCh: waitCh,
@@ -270,7 +270,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
}
// Open starts an rclone process with the given config.
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
be, err := newBackend(ctx, cfg, lim, errorLog)
if err != nil {
return nil, err
@@ -297,7 +297,7 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(st
}
// Create initializes a new restic repo with rclone.
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
be, err := newBackend(ctx, cfg, lim, errorLog)
if err != nil {
return nil, err
@@ -328,7 +328,7 @@ func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(
const waitForExit = 5 * time.Second
// Close terminates the backend.
func (be *Backend) Close() error {
func (be *rclone) Close() error {
debug.Log("exiting rclone")
be.tr.CloseIdleConnections()
@@ -348,7 +348,7 @@ func (be *Backend) Close() error {
return be.waitResult
}
func (be *Backend) Properties() backend.Properties {
func (be *rclone) Properties() backend.Properties {
properties := be.Backend.Properties()
properties.HasFlakyErrors = true
return properties
+1 -1
View File
@@ -27,7 +27,7 @@ func TestRcloneExit(t *testing.T) {
_ = be.Close()
}()
err = be.cmd.Process.Kill()
err = be.(*rclone).cmd.Process.Kill()
rtest.OK(t, err)
t.Log("killed rclone")
-1
View File
@@ -245,7 +245,6 @@ func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int,
}
if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 && resp.ContentLength != int64(length) {
_ = drainAndClose(resp)
return nil, &restError{h, http.StatusRequestedRangeNotSatisfiable, "partial out of bounds read"}
}
+23 -28
View File
@@ -25,15 +25,15 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
)
// Backend stores data on an S3 endpoint.
type Backend struct {
// s3 stores data on an S3 endpoint.
type s3 struct {
client *minio.Client
cfg Config
layout.Layout
}
// make sure that *Backend implements backend.Backend
var _ backend.Backend = &Backend{}
var _ backend.Backend = &s3{}
var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"}
@@ -50,7 +50,7 @@ func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
}
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
func open(cfg Config, rt http.RoundTripper) (*s3, error) {
debug.Log("open, config %#v", cfg)
if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) {
@@ -89,7 +89,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
return nil, errors.Wrap(err, "minio.New")
}
be := &Backend{
be := &s3{
client: client,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
@@ -240,12 +240,12 @@ func isAccessDenied(err error) bool {
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
func (be *s3) IsNotExist(err error) bool {
var e minio.ErrorResponse
return errors.As(err, &e) && e.Code == "NoSuchKey"
}
func (be *Backend) IsPermanentError(err error) bool {
func (be *s3) IsPermanentError(err error) bool {
if be.IsNotExist(err) {
return true
}
@@ -260,7 +260,7 @@ func (be *Backend) IsPermanentError(err error) bool {
return false
}
func (be *Backend) Properties() backend.Properties {
func (be *s3) Properties() backend.Properties {
return backend.Properties{
Connections: be.cfg.Connections,
HasAtomicReplace: true,
@@ -268,26 +268,21 @@ func (be *Backend) Properties() backend.Properties {
}
// Hasher may return a hash function for calculating a content hash for the backend
func (be *Backend) Hasher() hash.Hash {
func (be *s3) Hasher() hash.Hash {
return nil
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.cfg.Prefix
}
// useStorageClass returns whether file should be saved in the provided Storage Class
// For archive storage classes, only data files are stored using that class; metadata
// must remain instantly accessible.
func (be *Backend) useStorageClass(h backend.Handle) bool {
func (be *s3) useStorageClass(h backend.Handle) bool {
isDataFile := h.Type == backend.PackFile && !h.IsMetadata
isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass)
return !isArchiveClass || isDataFile
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
opts := minio.PutObjectOptions{
@@ -313,14 +308,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *s3) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
opts := minio.GetObjectOptions{}
@@ -352,7 +347,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
var obj *minio.Object
@@ -380,7 +375,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@@ -394,7 +389,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, recursive := be.Basedir(t)
// make sure prefix ends with a slash
@@ -449,15 +444,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
}
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
func (be *s3) Delete(ctx context.Context) error {
return util.DefaultDelete(ctx, be)
}
// Close does nothing
func (be *Backend) Close() error { return nil }
func (be *s3) Close() error { return nil }
// Warmup transitions handles from cold to hot storage if needed.
func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
handlesWarmingUp := []backend.Handle{}
if be.cfg.EnableRestore {
@@ -478,7 +473,7 @@ func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]back
}
// requestRestore sends a glacier restore request on a given file.
func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) {
func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error) {
objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{})
if err != nil {
return false, err
@@ -514,7 +509,7 @@ func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, e
}
// getWarmupStatus returns the warmup status of the provided object.
func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
// We can't use objectInfo.StorageClass to get the storage class of the
// object because this field is only set during ListObjects operations.
// The response header is the documented way to get the storage class
@@ -545,7 +540,7 @@ func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
}
// WarmupWait waits until all handles are in hot storage.
func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error {
func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout)
defer timeoutCtxCancel()
@@ -564,7 +559,7 @@ func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) err
}
// waitForRestore waits for a given file to be restored.
func (be *Backend) waitForRestore(ctx context.Context, filename string) error {
func (be *s3) waitForRestore(ctx context.Context, filename string) error {
for {
var objectInfo minio.ObjectInfo
+7 -39
View File
@@ -11,7 +11,6 @@ import (
"os"
"os/exec"
"path"
"sync/atomic"
"syscall"
"time"
@@ -38,8 +37,7 @@ type SFTP struct {
cmd *exec.Cmd
result <-chan error
posixRename bool
chmodBeforeRemove atomic.Bool
posixRename bool
layout.Layout
Config
@@ -365,7 +363,6 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
if err == nil {
err = f.Chmod(r.Modes.File)
if err != nil {
_ = f.Close()
return errors.Wrapf(err, "Chmod %v", tmpFilename)
}
}
@@ -407,11 +404,12 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
} else {
err = r.c.Rename(tmpFilename, filename)
}
if err != nil {
return errors.Wrapf(err, "Rename %v", tmpFilename)
}
err = setFileReadonly(r.c, filename, r.Modes.File)
return errors.Wrapf(err, "setFileReadonly %v", filename)
if err != nil {
return errors.Errorf("sftp setFileReadonly: %v", err)
}
return errors.Wrapf(err, "Rename %v", tmpFilename)
}
// checkNoSpace checks if err was likely caused by lack of available space
@@ -509,37 +507,7 @@ func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
return err
}
path := r.Filename(h)
if r.chmodBeforeRemove.Load() {
return r.removeWithChmod(path)
}
// optimistically try to remove the file
err := r.c.Remove(path)
if err == nil {
return nil
}
if !errors.Is(err, os.ErrPermission) {
return errors.Wrapf(err, "Remove %v", path)
}
// fallback to chmod + remove
// this is necessary on Windows where read-only files cannot be deleted without chmod.
if err := r.removeWithChmod(path); err != nil {
return err
}
r.chmodBeforeRemove.Store(true)
return nil
}
func (r *SFTP) removeWithChmod(path string) error {
err := r.c.Chmod(path, r.Modes.File)
if err != nil {
return errors.Wrapf(err, "Chmod %v", path)
}
return errors.Wrapf(r.c.Remove(path), "Remove %v", path)
return errors.Wrapf(r.c.Remove(r.Filename(h)), "Remove %v", r.Filename(h))
}
// List runs fn for each file in the backend which has the type t. When an
+28 -47
View File
@@ -251,10 +251,6 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
test.OK(t, b.Remove(context.TODO(), handle))
}
type setter interface {
SetListMaxItems(int)
}
// TestList makes sure that the backend implements List() pagination correctly.
func (s *Suite[C]) TestList(t *testing.T) {
random := seedRand(t)
@@ -302,54 +298,39 @@ func (s *Suite[C]) TestList(t *testing.T) {
t.Logf("wrote %v files", len(list1))
var tests = []struct {
maxItems int
}{
{11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123},
list2 := make(map[restic.ID]int64)
err = b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
}
list2[id] = fi.Size
return nil
})
if err != nil {
t.Fatalf("List returned error %v", err)
}
for _, test := range tests {
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
list2 := make(map[restic.ID]int64)
t.Logf("loaded %v IDs from backend", len(list2))
if s, ok := b.(setter); ok {
t.Logf("setting max list items to %d", test.maxItems)
s.SetListMaxItems(test.maxItems)
}
for id, size := range list1 {
size2, ok := list2[id]
if !ok {
t.Errorf("id %v not returned by List()", id.Str())
}
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
}
list2[id] = fi.Size
return nil
})
if size != size2 {
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
}
}
if err != nil {
t.Fatalf("List returned error %v", err)
}
t.Logf("loaded %v IDs from backend", len(list2))
for id, size := range list1 {
size2, ok := list2[id]
if !ok {
t.Errorf("id %v not returned by List()", id.Str())
}
if size != size2 {
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
}
}
for id := range list2 {
_, ok := list1[id]
if !ok {
t.Errorf("extra id %v returned by List()", id.Str())
}
}
})
for id := range list2 {
_, ok := list1[id]
if !ok {
t.Errorf("extra id %v returned by List()", id.Str())
}
}
t.Logf("remove %d files", numTestFiles)
+2 -7
View File
@@ -35,15 +35,10 @@ func DefaultDelete(ctx context.Context, be backend.Backend) error {
for _, t := range alltypes {
err := be.List(ctx, t, func(fi backend.FileInfo) error {
err := be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
if err != nil && be.IsNotExist(err) {
// deletion of files created by TestSaveError may happen with a delay for the REST server, so we ignore the error
err = nil
}
return err
return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
})
if err != nil {
return err
return nil
}
}
err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})
+30 -59
View File
@@ -4,7 +4,6 @@ import (
"context"
"io"
"math/rand"
"os"
"path/filepath"
"sort"
"strconv"
@@ -19,7 +18,6 @@ import (
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/hashing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@@ -75,7 +73,8 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) {
}
func TestCheckRepo(t *testing.T) {
repo, _ := repository.TestFromFixture(t, checkerTestData)
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -92,7 +91,8 @@ func TestCheckRepo(t *testing.T) {
}
func TestMissingPack(t *testing.T) {
repo, be := repository.TestFromFixture(t, checkerTestData)
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
packID := restic.TestParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6")
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: packID.String()}))
@@ -109,7 +109,7 @@ func TestMissingPack(t *testing.T) {
test.Assert(t, len(errs) == 1,
"expected exactly one error, got %v", len(errs))
if err, ok := errs[0].(*repository.PackError); ok {
if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
test.Equals(t, packID, err.ID)
} else {
t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -117,7 +117,8 @@ func TestMissingPack(t *testing.T) {
}
func TestUnreferencedPack(t *testing.T) {
repo, be := repository.TestFromFixture(t, checkerTestData)
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
// index 3f1a only references pack 60e0
packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
@@ -136,7 +137,7 @@ func TestUnreferencedPack(t *testing.T) {
test.Assert(t, len(errs) == 1,
"expected exactly one error, got %v", len(errs))
if err, ok := errs[0].(*repository.PackError); ok {
if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
test.Equals(t, packID, err.ID.String())
} else {
t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -144,7 +145,8 @@ func TestUnreferencedPack(t *testing.T) {
}
func TestUnreferencedBlobs(t *testing.T) {
repo, be := repository.TestFromFixture(t, checkerTestData)
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
snapshotID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.SnapshotFile, Name: snapshotID.String()}))
@@ -178,7 +180,8 @@ func TestUnreferencedBlobs(t *testing.T) {
}
func TestModifiedIndex(t *testing.T) {
repo, be := repository.TestFromFixture(t, checkerTestData)
repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
done := make(chan struct{})
defer close(done)
@@ -187,56 +190,19 @@ func TestModifiedIndex(t *testing.T) {
Type: restic.IndexFile,
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
tmpfile, err := os.CreateTemp("", "restic-test-mod-index-")
if err != nil {
t.Fatal(err)
}
defer func() {
err := tmpfile.Close()
if err != nil {
t.Fatal(err)
}
err = os.Remove(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
}()
wr := io.Writer(tmpfile)
var hw *hashing.Writer
if be.Hasher() != nil {
hw = hashing.NewWriter(wr, be.Hasher())
wr = hw
}
// read the file from the backend
err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
_, err := io.Copy(wr, rd)
var data []byte
test.OK(t, be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
var err error
data, err = io.ReadAll(rd)
return err
})
test.OK(t, err)
}))
// save the index again with a modified name so that the hash doesn't match
// the content any more
h2 := backend.Handle{
Type: restic.IndexFile,
Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
var hash []byte
if hw != nil {
hash = hw.Sum(nil)
}
rd, err := backend.NewFileReader(tmpfile, hash)
if err != nil {
t.Fatal(err)
}
err = be.Save(context.TODO(), h2, rd)
if err != nil {
t.Fatal(err)
}
test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher())))
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -254,7 +220,8 @@ func TestModifiedIndex(t *testing.T) {
var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
func TestDuplicatePacksInIndex(t *testing.T) {
repo, _ := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
repo, _, cleanup := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
defer cleanup()
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -453,7 +420,8 @@ func (r *loadTreesOnceRepository) LoadBlob(ctx context.Context, t restic.BlobTyp
}
func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
repo, _ := repository.TestFromFixture(t, checkerTestData)
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
defer cleanup()
checkRepo := &loadTreesOnceRepository{
Repository: repo,
loadedTrees: restic.NewIDSet(),
@@ -601,12 +569,13 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
test.Assert(t, delayRepo.Triggered, "delay repository did not trigger")
}
func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository) {
repo, _ := repository.TestFromFixture(t, checkerTestData)
func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, func()) {
repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
if len(errs) > 0 {
defer cleanup()
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
}
@@ -615,11 +584,12 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository) {
t.Fatalf("expected mixed pack hint, got %v", err)
}
}
return chkr, repo
return chkr, repo, cleanup
}
func BenchmarkChecker(t *testing.B) {
chkr, _ := loadBenchRepository(t)
chkr, _, cleanup := loadBenchRepository(t)
defer cleanup()
t.ResetTimer()
@@ -631,7 +601,8 @@ func BenchmarkChecker(t *testing.B) {
}
func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) {
chkr, repo := loadBenchRepository(t)
chkr, repo, cleanup := loadBenchRepository(t)
defer cleanup()
snID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
sn2, err := data.LoadSnapshot(context.TODO(), repo, snID)
+1 -1
View File
@@ -109,7 +109,7 @@ func nodeFillExtendedAttributes(node *data.Node, path string, ignoreListError bo
for _, attr := range xattrs {
attrVal, err := getxattr(path, attr)
if err != nil {
warnf("can not obtain extended attribute %v for %v: %v\n", attr, path, err)
warnf("can not obtain extended attribute %v for %v:\n", attr, path)
continue
}
attr := data.ExtendedAttribute{
+1 -1
View File
@@ -225,7 +225,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
return nil, err
}
return d.cache.lookupOrCreate(name, -1, func(forget forgetFn) (fs.Node, error) {
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
node, ok := d.items[name]
if !ok {
debug.Log(" Lookup(%v) -> not found", name)
-33
View File
@@ -247,39 +247,6 @@ func TestStableNodeObjects(t *testing.T) {
testStableLookup(t, dir, "file-2")
}
func TestSnapshotsDirLatestSymlinkUpdatesAfterReload(t *testing.T) {
repo := repository.TestRepository(t)
timeTemplate := "2006-01-02T15:04:05"
firstTime, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56Z")
rtest.OK(t, err)
secondTime := firstTime.Add(time.Hour)
data.TestCreateSnapshot(t, repo, firstTime, 0)
root := NewRoot(repo, Config{
TimeTemplate: timeTemplate,
PathTemplates: []string{"snapshots/%T"},
})
snapshotsDir := testStableLookup(t, root, "snapshots")
rtest.Equals(t, firstTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
data.TestCreateSnapshot(t, repo, secondTime, 0)
root.SnapshotsDir.dirStruct.lastCheck = time.Now().Add(-2 * minSnapshotsReloadTime)
rtest.Equals(t, secondTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
}
func readLatestTarget(t testing.TB, node fs.Node) string {
t.Helper()
latest, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), "latest")
rtest.OK(t, err)
target, err := latest.(fs.NodeReadlinker).Readlink(context.TODO(), nil)
rtest.OK(t, err)
return target
}
// Test reporting of fuse.Attr.Blocks in multiples of 512.
func TestBlocks(t *testing.T) {
root := &Root{}
+3 -3
View File
@@ -61,7 +61,7 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("ReadDirAll()")
// update snapshots
meta, _, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
return nil, unwrapCtxCanceled(err)
} else if meta == nil {
@@ -104,14 +104,14 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("Lookup(%s)", name)
meta, gen, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
return nil, unwrapCtxCanceled(err)
} else if meta == nil {
return nil, syscall.ENOENT
}
return d.cache.lookupOrCreate(name, gen, func(forget forgetFn) (fs.Node, error) {
return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
entry := meta.names[name]
if entry == nil {
return nil, syscall.ENOENT
+3 -8
View File
@@ -42,10 +42,6 @@ type SnapshotsDirStructure struct {
hash [sha256.Size]byte // Hash at last check.
lastCheck time.Time
// generation is incremented whenever the directory structure is rebuilt.
// It allows treeCache instances to detect stale entries and reset themselves.
generation int64
}
// NewSnapshotsDirStructure returns a new directory structure for snapshots.
@@ -284,7 +280,6 @@ func (d *SnapshotsDirStructure) makeDirs(snapshots data.Snapshots) {
}
d.entries = entries
d.generation++
}
const minSnapshotsReloadTime = 60 * time.Second
@@ -342,13 +337,13 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
return nil
}
func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, int64, error) {
func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) {
err := d.updateSnapshots(ctx)
if err != nil {
return nil, 0, err
return nil, err
}
d.mutex.Lock()
defer d.mutex.Unlock()
return d.entries[prefix], d.generation, nil
return d.entries[prefix], nil
}
+4 -16
View File
@@ -5,15 +5,12 @@ package fuse
import (
"sync"
"github.com/restic/restic/internal/debug"
"github.com/anacrolix/fuse/fs"
)
type treeCache struct {
nodes map[string]fs.Node
m sync.Mutex
generation int64
nodes map[string]fs.Node
m sync.Mutex
}
type forgetFn func()
@@ -24,28 +21,19 @@ func newTreeCache() *treeCache {
}
}
func (t *treeCache) lookupOrCreate(name string, generation int64, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
t.m.Lock()
defer t.m.Unlock()
if generation >= 0 && generation != t.generation {
debug.Log("treeCache generation changed %d -> %d, resetting cache", t.generation, generation)
t.nodes = make(map[string]fs.Node)
t.generation = generation
}
if node, ok := t.nodes[name]; ok {
return node, nil
}
cacheGeneration := t.generation
node, err := create(func() {
t.m.Lock()
defer t.m.Unlock()
if t.generation == cacheGeneration {
delete(t.nodes, name)
}
delete(t.nodes, name)
})
if err != nil {
return nil, err
-82
View File
@@ -1,82 +0,0 @@
//go:build darwin || freebsd || linux
package fuse
import (
"context"
"testing"
"github.com/anacrolix/fuse"
"github.com/anacrolix/fuse/fs"
"github.com/restic/restic/internal/test"
)
type cacheTestNode struct {
id int
}
func (n cacheTestNode) Attr(context.Context, *fuse.Attr) error {
return nil
}
func TestTreeCacheGeneration(t *testing.T) {
cache := newTreeCache()
created := 0
create := func(forgetFn) (fs.Node, error) {
created++
return cacheTestNode{id: created}, nil
}
node1, err := cache.lookupOrCreate("node", 1, create)
test.OK(t, err)
node2, err := cache.lookupOrCreate("node", 1, create)
test.OK(t, err)
test.Assert(t, node1 == node2, "lookup should reuse cached node")
test.Equals(t, 1, created)
node3, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node1 != node3, "lookup should recreate node after generation change")
test.Equals(t, 2, created)
node4, err := cache.lookupOrCreate("node", -1, create)
test.OK(t, err)
test.Assert(t, node3 == node4, "negative generation should not reset cache")
test.Equals(t, 2, created)
node5, err := cache.lookupOrCreate("node", 3, create)
test.OK(t, err)
test.Assert(t, node4 != node5, "lookup should still track later generation changes")
test.Equals(t, 3, created)
}
func TestTreeCacheForgetOnlyRemovesSameGeneration(t *testing.T) {
cache := newTreeCache()
created := 0
var forgets []forgetFn
create := func(forget forgetFn) (fs.Node, error) {
created++
forgets = append(forgets, forget)
return cacheTestNode{id: created}, nil
}
node1, err := cache.lookupOrCreate("node", 1, create)
test.OK(t, err)
node2, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node1 != node2, "lookup should recreate node after generation change")
test.Equals(t, 2, created)
forgets[0]()
node3, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node2 == node3, "forget from an old generation must not remove the current node")
test.Equals(t, 2, created)
forgets[1]()
node4, err := cache.lookupOrCreate("node", 2, create)
test.OK(t, err)
test.Assert(t, node3 != node4, "forget from the current generation should remove the cached node")
test.Equals(t, 3, created)
}
+1 -1
View File
@@ -35,7 +35,7 @@ import (
// to a missing backend storage location or config file
var ErrNoRepository = errors.New("repository does not exist")
const Version = "0.19.1"
const Version = "0.19.0-dev (compiled manually)"
// TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05"
-215
View File
@@ -1,215 +0,0 @@
package repository
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"fmt"
"io"
"github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/hashing"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
)
// ErrPackData is returned if errors are discovered while verifying a packfile
type ErrPackData struct {
PackID restic.ID
errs []error
}
func (e *ErrPackData) Error() string {
return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
}
type partialReadError struct {
err error
}
func (e *partialReadError) Error() string {
return e.err.Error()
}
// CheckPack reads a pack and checks the integrity of all blobs.
func CheckPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err != nil {
if r.cache != nil {
// ignore error as there's not much we can do here
_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
}
// retry pack verification to detect transient errors
err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err2 != nil {
err = err2
} else {
err = fmt.Errorf("check successful on second attempt, original error %w", err)
}
}
return err
}
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
debug.Log("checking pack %v", id.String())
if len(blobs) == 0 {
return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
}
// sanity check blobs in index
blobs.Sort()
idxHdrSize := pack.CalculateHeaderSize(blobs)
lastBlobEnd := 0
nonContinuousPack := false
for _, blob := range blobs {
if lastBlobEnd != int(blob.Offset) {
nonContinuousPack = true
}
lastBlobEnd = int(blob.Offset + blob.Length)
}
// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
var errs []error
if nonContinuousPack {
debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
}
// calculate hash on-the-fly while reading the pack and capture pack header
var hash restic.ID
var hdrBuf []byte
// must use a separate slice from `errs` here as we're only interested in the last retry
var blobErrors []error
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
hrd := hashing.NewReader(rd, sha256.New())
bufRd.Reset(hrd)
// reset blob errors for each retry
blobErrors = nil
it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
for {
if ctx.Err() != nil {
return ctx.Err()
}
val, err := it.Next()
if err == errPackEOF {
break
} else if err != nil {
return &partialReadError{err}
}
debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle)
if val.Err != nil {
debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err)
blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
}
}
// skip enough bytes until we reach the possible header start
curPos := lastBlobEnd
minHdrStart := int(size) - pack.MaxHeaderSize
if minHdrStart > curPos {
_, err := bufRd.Discard(minHdrStart - curPos)
if err != nil {
return &partialReadError{err}
}
curPos += minHdrStart - curPos
}
// read remainder, which should be the pack header
var err error
hdrBuf = make([]byte, int(size-int64(curPos)))
_, err = io.ReadFull(bufRd, hdrBuf)
if err != nil {
return &partialReadError{err}
}
hash = restic.IDFromHash(hrd.Sum(nil))
return nil
})
errs = append(errs, blobErrors...)
if err != nil {
var e *partialReadError
isPartialReadError := errors.As(err, &e)
// failed to load the pack file, return as further checks cannot succeed anyways
debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err)
if isPartialReadError {
return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
}
// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
// completely failed to download such that there's no point in repairing anything.
return fmt.Errorf("download error: %w", err)
}
if !hash.Equal(id) {
debug.Log("pack ID does not match, want %v, got %v", id, hash)
return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
}
blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
if err != nil {
return &ErrPackData{PackID: id, errs: append(errs, err)}
}
if uint32(idxHdrSize) != hdrSize {
debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
}
for _, blob := range blobs {
// Check if blob is contained in index and position is correct
idxHas := false
for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
if pb.PackID == id && pb.Blob == blob {
idxHas = true
break
}
}
if !idxHas {
errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
continue
}
}
if len(errs) > 0 {
return &ErrPackData{PackID: id, errs: errs}
}
return nil
}
type bufReader struct {
rd *bufio.Reader
buf []byte
}
func newBufReader(rd *bufio.Reader) *bufReader {
return &bufReader{
rd: rd,
}
}
func (b *bufReader) Discard(n int) (discarded int, err error) {
return b.rd.Discard(n)
}
func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
if cap(b.buf) < n {
b.buf = make([]byte, n)
}
b.buf = b.buf[:n]
_, err = io.ReadFull(b.rd, b.buf)
if err != nil {
return nil, err
}
return b.buf, nil
}
+207 -8
View File
@@ -2,12 +2,17 @@ package repository
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"fmt"
"io"
"github.com/klauspost/compress/zstd"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/hashing"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
@@ -46,18 +51,29 @@ func (e *ErrMixedPack) Error() string {
return fmt.Sprintf("pack %v contains a mix of tree and data blobs", e.PackID.Str())
}
// PackError describes an error with a specific pack.
type PackError struct {
// ErrPackMetadata describes an error with a specific pack. It is used for missing, truncated or orphaned packs.
// Errors of the actual pack data are returned as ErrPackData.
type ErrPackMetadata struct {
ID restic.ID
Orphaned bool
Truncated bool
Err error
}
func (e *PackError) Error() string {
func (e *ErrPackMetadata) Error() string {
return "pack " + e.ID.String() + ": " + e.Err.Error()
}
// ErrPackData is returned if errors are discovered while verifying a packfile
type ErrPackData struct {
PackID restic.ID
errs []error
}
func (e *ErrPackData) Error() string {
return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
}
// Checker handles index-related operations for repository checking.
type Checker struct {
repo *Repository
@@ -199,7 +215,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
select {
case <-ctx.Done():
return
case errChan <- &PackError{ID: id, Err: errors.New("does not exist")}:
case errChan <- &ErrPackMetadata{ID: id, Err: errors.New("does not exist")}:
}
continue
}
@@ -209,7 +225,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
select {
case <-ctx.Done():
return
case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
case errChan <- &ErrPackMetadata{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
}
}
}
@@ -219,7 +235,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
select {
case <-ctx.Done():
return
case errChan <- &PackError{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
case errChan <- &ErrPackMetadata{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
}
}
}
@@ -253,7 +269,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize)
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
panic(dec)
}
defer dec.Close()
for {
@@ -269,7 +285,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
}
}
err := CheckPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
p.Add(1)
if err == nil {
continue
@@ -309,3 +325,186 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
}
}
}
// checkPack reads a pack and checks the integrity of all blobs.
func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err != nil {
if r.cache != nil {
// ignore error as there's not much we can do here
_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
}
// retry pack verification to detect transient errors
err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
if err2 != nil {
err = err2
} else {
err = fmt.Errorf("check successful on second attempt, original error %w", err)
}
}
return err
}
func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
type partialReadError struct {
error
}
debug.Log("checking pack %v", id.String())
if len(blobs) == 0 {
return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
}
// sanity check blobs in index
blobs.Sort()
idxHdrSize := pack.CalculateHeaderSize(blobs)
lastBlobEnd := 0
nonContinuousPack := false
for _, blob := range blobs {
if lastBlobEnd != int(blob.Offset) {
nonContinuousPack = true
}
lastBlobEnd = int(blob.Offset + blob.Length)
}
// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
var errs []error
if nonContinuousPack {
debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
}
// calculate hash on-the-fly while reading the pack and capture pack header
var hash restic.ID
var hdrBuf []byte
// must use a separate slice from `errs` here as we're only interested in the last retry
var blobErrors []error
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
hrd := hashing.NewReader(rd, sha256.New())
bufRd.Reset(hrd)
// reset blob errors for each retry
blobErrors = nil
it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
for {
if ctx.Err() != nil {
return ctx.Err()
}
val, err := it.Next()
if err == errPackEOF {
break
} else if err != nil {
return &partialReadError{err}
}
debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle)
if val.Err != nil {
debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err)
blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
}
}
// skip enough bytes until we reach the possible header start
curPos := lastBlobEnd
minHdrStart := int(size) - pack.MaxHeaderSize
if minHdrStart > curPos {
_, err := bufRd.Discard(minHdrStart - curPos)
if err != nil {
return &partialReadError{err}
}
curPos += minHdrStart - curPos
}
// read remainder, which should be the pack header
var err error
hdrBuf = make([]byte, int(size-int64(curPos)))
_, err = io.ReadFull(bufRd, hdrBuf)
if err != nil {
return &partialReadError{err}
}
hash = restic.IDFromHash(hrd.Sum(nil))
return nil
})
errs = append(errs, blobErrors...)
if err != nil {
var e *partialReadError
isPartialReadError := errors.As(err, &e)
// failed to load the pack file, return as further checks cannot succeed anyways
debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err)
if isPartialReadError {
return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
}
// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
// completely failed to download such that there's no point in repairing anything.
return fmt.Errorf("download error: %w", err)
}
if !hash.Equal(id) {
debug.Log("pack ID does not match, want %v, got %v", id, hash)
return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
}
blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
if err != nil {
return &ErrPackData{PackID: id, errs: append(errs, err)}
}
if uint32(idxHdrSize) != hdrSize {
debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
}
for _, blob := range blobs {
// Check if blob is contained in index and position is correct
idxHas := false
for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
if pb.PackID == id && pb.Blob == blob {
idxHas = true
break
}
}
if !idxHas {
errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
continue
}
}
if len(errs) > 0 {
return &ErrPackData{PackID: id, errs: errs}
}
return nil
}
type bufReader struct {
rd *bufio.Reader
buf []byte
}
func newBufReader(rd *bufio.Reader) *bufReader {
return &bufReader{
rd: rd,
}
}
func (b *bufReader) Discard(n int) (discarded int, err error) {
return b.rd.Discard(n)
}
func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
if cap(b.buf) < n {
b.buf = make([]byte, n)
}
b.buf = b.buf[:n]
_, err = io.ReadFull(b.rd, b.buf)
if err != nil {
return nil, err
}
return b.buf, nil
}
+390
View File
@@ -0,0 +1,390 @@
//go:build debug
package repository
import (
"context"
"crypto/aes"
"crypto/cipher"
"encoding/json"
"fmt"
"io"
"os"
"runtime"
"sync"
"time"
"github.com/klauspost/compress/zstd"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
)
type packDumpEntry struct {
Name string `json:"name"`
Blobs []packDumpBlob `json:"blobs"`
}
type packDumpBlob struct {
Type restic.BlobType `json:"type"`
Length uint `json:"length"`
ID restic.ID `json:"id"`
Offset uint `json:"offset"`
}
func writePackDumpJSON(wr io.Writer, item any) error {
buf, err := json.MarshalIndent(item, "", " ")
if err != nil {
return err
}
_, err = wr.Write(append(buf, '\n'))
return err
}
// DumpPacks lists each pack file and writes its header blob layout as JSON to wr.
func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, err := repo.ListPack(ctx, id, size)
if err != nil {
printer.E("error for pack %v: %v", id.Str(), err)
return nil
}
p := packDumpEntry{
Name: id.String(),
Blobs: make([]packDumpBlob, len(blobs)),
}
for i, blob := range blobs {
p.Blobs[i] = packDumpBlob{
Type: blob.Type,
Length: blob.Length,
ID: blob.ID,
Offset: blob.Offset,
}
}
m.Lock()
defer m.Unlock()
return writePackDumpJSON(wr, p)
})
}
// DumpIndexes loads each on-disk index file and writes its debug dump to wr.
func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id)
if err != nil {
return err
}
return idx.Dump(wr)
})
}
// ExaminePackOptions configures debug examination of a pack file.
type ExaminePackOptions struct {
TryRepair bool
RepairByte bool
ExtractPack bool
ReuploadBlobs bool
}
// ExaminePack loads and inspects a pack file and its index entries.
func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error {
printer.S("examine %v", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
printer.S(" file size is %v", len(buf))
gotID := restic.Hash(buf)
if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID)
} else {
printer.S(" hash for file content matches")
}
printer.S(" ========================================")
printer.S(" looking for info in the indexes")
blobsLoaded := false
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
continue
}
checkPackSize(blobs, len(buf), printer)
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
if err != nil {
printer.E("error: %v", err)
} else {
blobsLoaded = true
}
}
printer.S(" ========================================")
printer.S(" inspect the pack itself")
blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
checkPackSize(blobs, len(buf), printer)
if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer)
}
return nil
}
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
// track current size and offset
var size, offset uint64
blobs.Sort()
for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
}
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
}
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
} else {
printer.S(" file sizes match")
}
}
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
if bytewise {
printer.S(" trying to repair blob by finding a broken byte")
} else {
printer.S(" trying to repair blob with single bit flip")
}
ch := make(chan int)
var wg errgroup.Group
done := make(chan struct{})
var fixed []byte
var found bool
workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ {
wg.Go(func() error {
// make a local copy of the buffer
buf := make([]byte, len(input))
copy(buf, input)
testFlip := func(idx int, pattern byte) bool {
// flip bits
buf[idx] ^= pattern
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil {
printer.S("")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext))
close(done)
found = true
fixed = plaintext
return true
}
// flip bits back
buf[idx] ^= pattern
return false
}
for i := range ch {
if bytewise {
for j := 0; j < 255; j++ {
if testFlip(i, byte(j)) {
return nil
}
}
} else {
for j := 0; j < 7; j++ {
// flip each bit once
if testFlip(i, (1 << uint(j))) {
return nil
}
}
}
}
return nil
})
}
wg.Go(func() error {
defer close(ch)
start := time.Now()
info := time.Now()
for i := range input {
select {
case ch <- i:
case <-done:
printer.S(" done after %v", time.Since(start))
return nil
}
if time.Since(info) > time.Second {
secs := time.Since(start).Seconds()
gps := float64(i) / secs
remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now()
}
}
return nil
})
err := wg.Wait()
if err != nil {
panic("all go routines can only return nil")
}
if !found {
printer.S("\n blob could not be repaired")
}
return fixed
}
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
// strip signature at the end
l := len(buf)
nonce, ct := buf[:16], buf[16:l-16]
out := make([]byte, len(ct))
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCTR(c, nonce)
e.XORKeyStream(out, ct)
return out
}
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
packData, err := repo.LoadRaw(ctx, restic.PackFile, packID)
// allow processing broken pack files
if packData == nil {
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(packData) {
printer.E("skipping truncated blob")
continue
}
buf := packData[blob.Offset : blob.Offset+blob.Length]
key := repo.Key()
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
outputPrefix := ""
filePrefix := ""
if err != nil {
printer.E("error decrypting blob: %v", err)
if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
if err != nil {
return err
}
continue
}
}
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
printer.S(" failed to decompress blob %v", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
}
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
if err != nil {
return err
}
}
if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
printer.S(" uploaded %v %v", blob.Type, id)
}
}
return nil
})
return err
}
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename)
if err != nil {
return err
}
_, err = f.Write(plain)
if err != nil {
_ = f.Close()
return err
}
err = f.Close()
if err != nil {
return err
}
printer.S("decrypt of blob %v stored at %v", id, filename)
return nil
}
@@ -15,7 +15,8 @@ import (
var repoFixture = filepath.Join("..", "testdata", "test-repo.tar.gz")
func TestRepositoryForAllIndexes(t *testing.T) {
repo, _ := repository.TestFromFixture(t, repoFixture)
repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
defer cleanup()
expectedIndexIDs := restic.NewIDSet()
rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
+1 -1
View File
@@ -280,7 +280,6 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
}
func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
defer p.Done()
indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
if err != nil {
return err
@@ -303,6 +302,7 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
return err
}
p.SetMax(numIndexFiles)
defer p.Done()
}
err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error {
@@ -694,3 +694,62 @@ func TestRewriteSplitPacks(t *testing.T) {
blobs := mi.Lookup(blobOther.BlobHandle)
rtest.Equals(t, nil, blobs)
}
// TestRewriteFullPacks checks that Rewrite drops a duplicate full index for the same
// pack while keeping the other index files and blob lookups intact. Creates 3 indexes:
// - indexA: contains packA
// - indexB: contains packB
// - indexC: contains packB
// After the rewrite, indexC must be dropped. The other indexes must be kept.
func TestRewriteFullPacks(t *testing.T) {
originalFull := index.Full
defer func() {
index.Full = originalFull
}()
index.Full = func(*index.Index) bool { return true }
repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion)
packA := restic.NewRandomID()
packB := restic.NewRandomID()
blobA := restic.PackedBlob{
PackID: packA,
Blob: restic.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
blobB := restic.PackedBlob{
PackID: packB,
Blob: restic.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(50)),
Offset: 0,
},
}
mi := index.NewMasterIndex()
rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
indexIDs := mi.IDs()
rtest.Equals(t, 3, len(indexIDs))
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
mi2 := index.NewMasterIndex()
rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
afterRewrite := mi2.IDs()
rtest.Equals(t, 2, len(afterRewrite))
rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
}
+41
View File
@@ -0,0 +1,41 @@
package repository
import (
"context"
"iter"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
)
// IndexBlob is one blob handle from an on-disk index file, or an error from loading/decoding
// that file.
type IndexBlob struct {
Handle restic.BlobHandle
Error error
}
// AllIndexBlobs streams blob handles from each index file without building a master index.
func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.LoaderUnpacked) iter.Seq[IndexBlob] {
return func(yield func(IndexBlob) bool) {
stopIteration := errors.New("stop index blob iteration")
err := index.ForAllIndexes(ctx, lister, loader, func(_ restic.ID, idx *index.Index, err error) error {
if err != nil {
return err
}
for blob := range idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
}
if !yield(IndexBlob{Handle: blob.BlobHandle}) {
return stopIteration
}
}
return nil
})
if err != nil && !errors.Is(err, stopIteration) {
yield(IndexBlob{Error: err})
}
}
}
+62
View File
@@ -0,0 +1,62 @@
package repository_test
import (
"context"
"testing"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestAllIndexBlobs(t *testing.T) {
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
want := restic.NewBlobSet()
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for i := range 5 {
data := []byte{byte('a' + i)}
id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, data, restic.ID{}, false)
rtest.OK(t, err)
want.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id})
}
return nil
}))
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
fromMaster := restic.NewBlobSet()
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
fromMaster.Insert(pb.BlobHandle)
}))
rtest.Equals(t, want, fromMaster)
fromStream := restic.NewBlobSet()
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
if entry.Error != nil {
t.Fatalf("unexpected error: %v", entry.Error)
}
fromStream.Insert(entry.Handle)
}
rtest.Equals(t, want, fromStream)
}
func TestAllIndexBlobsEarlyStop(t *testing.T) {
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for range 5 {
_, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, []byte("test"), restic.ID{}, false)
rtest.OK(t, err)
}
return nil
}))
var count int
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
rtest.Assert(t, entry.Error == nil, "unexpected error after early stop: %v", entry.Error)
count++
break
}
rtest.Equals(t, 1, count)
}
-2
View File
@@ -586,7 +586,6 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
// unreferenced packs can be safely deleted first
if len(plan.removePacksFirst) != 0 {
printer.P("deleting unreferenced packs\n")
// ignoring errors is fine here as keeping too many packs cannot damage the repository
_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacksFirst, restic.PackFile, printer)
// forget unused data
plan.removePacksFirst = nil
@@ -644,7 +643,6 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
if len(plan.removePacks) != 0 {
printer.P("removing %d old packs", len(plan.removePacks))
// ignoring errors is fine here as keeping too many packs cannot damage the repository
_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacks, restic.PackFile, printer)
}
if ctx.Err() != nil {
+1 -1
View File
@@ -86,7 +86,7 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
blobs := restic.NewBlobSet()
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
entries, _, err := repo.ListPack(context.TODO(), id, size)
entries, err := repo.ListPack(context.TODO(), id, size)
if err != nil {
t.Fatalf("error listing pack %v: %v", id, err)
}
+1 -2
View File
@@ -76,9 +76,8 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
if ids.Has(id) {
blobs, _, err := repo.ListPack(ctx, id, size)
blobs, err := repo.ListPack(ctx, id, size)
if err != nil {
// ignore errors for broken pack files to be able to salvage as much as possible
return nil
}
packToBlobs[id] = blobs
+12 -8
View File
@@ -783,7 +783,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
worker := func() error {
for fi := range ch {
entries, _, err := r.ListPack(wgCtx, fi.ID, fi.Size)
entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
if err != nil {
debug.Log("unable to list pack file %v", fi.ID.Str())
m.Lock()
@@ -844,8 +844,13 @@ func (r *Repository) prepareCache() error {
packs := r.idx.Packs(restic.NewIDSet())
ids := make(map[string]struct{})
for id := range packs {
ids[id.String()] = struct{}{}
}
// clear old packs
return r.cache.Clear(restic.PackFile, packs)
return r.cache.Clear(backend.PackFile, ids)
}
// SearchKey finds a key with the supplied password, afterwards the config is
@@ -959,12 +964,11 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
})
}
// ListPack returns the list of blobs saved in the pack id and the length of
// the pack header.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, uint32, error) {
// ListPack returns the list of blobs saved in the pack id.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
if err != nil {
if r.cache != nil {
// ignore error as there is not much we can do here
@@ -972,9 +976,9 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
}
// retry on error
entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
}
return entries, hdrSize, err
return entries, err
}
// Delete calls backend.Delete() if implemented, and returns an error
+5 -3
View File
@@ -319,7 +319,8 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
var repoFixture = filepath.Join("testdata", "test-repo.tar.gz")
func TestRepositoryLoadIndex(t *testing.T) {
repo, _ := repository.TestFromFixture(t, repoFixture)
repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
defer cleanup()
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
}
@@ -372,7 +373,8 @@ func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length
}
func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
repodir := rtest.Env(t, repoFixture)
repodir, cleanup := rtest.Env(t, repoFixture)
defer cleanup()
be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2}, t.Logf)
rtest.OK(t, err)
@@ -477,7 +479,7 @@ func TestListPack(t *testing.T) {
return nil
}))
blobs, _, err := repo.ListPack(context.TODO(), packID, size)
blobs, err := repo.ListPack(context.TODO(), packID, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
+3 -3
View File
@@ -105,11 +105,11 @@ func TestRepositoryWithVersion(t testing.TB, version uint) (*Repository, restic.
return repo, &internalRepository{repo}, be
}
func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend) {
repodir := test.Env(t, repoFixture)
func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend, func()) {
repodir, cleanup := test.Env(t, repoFixture)
repo, be := TestOpenLocal(t, repodir)
return repo, be
return repo, be, cleanup
}
// TestOpenLocal opens a local repository.
+2 -2
View File
@@ -24,13 +24,13 @@ func (e *NoIDByPrefixError) Error() string {
// Find loads the list of all files of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned.
func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
func Find(ctx context.Context, repo Lister, t FileType, prefix string) (ID, error) {
match := ID{}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := be.List(ctx, t, func(id ID, _ int64) error {
err := repo.List(ctx, t, func(id ID, _ int64) error {
name := id.String()
if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
if match.IsNull() {
-4
View File
@@ -57,10 +57,6 @@ func (h BlobHandle) String() string {
return fmt.Sprintf("<%s/%s>", h.Type, h.ID.Str())
}
func NewRandomBlobHandle() BlobHandle {
return BlobHandle{ID: NewRandomID(), Type: DataBlob}
}
// BlobType specifies what a blob stored in a pack is.
type BlobType uint8
-5
View File
@@ -26,11 +26,6 @@ const MaxRepoVersion = 2
// is newly created with Init().
const StableRepoVersion = 2
// JSONUnpackedLoader loads unpacked JSON.
type JSONUnpackedLoader interface {
LoadJSONUnpacked(context.Context, FileType, ID, interface{}) error
}
// CreateConfig creates a config file with a randomly selected polynomial and
// ID.
func CreateConfig(version uint) (Config, error) {
-13
View File
@@ -1,11 +1,9 @@
package restic
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
)
// Hash returns the ID for data.
@@ -40,17 +38,6 @@ func (id ID) String() string {
return hex.EncodeToString(id[:])
}
// NewRandomID returns a randomly generated ID. When reading from rand fails,
// the function panics.
func NewRandomID() ID {
id := ID{}
_, err := io.ReadFull(rand.Reader, id[:])
if err != nil {
panic(err)
}
return id
}
const shortStr = 4
// Str returns the shortened string version of id.
+2 -3
View File
@@ -32,9 +32,8 @@ type Repository interface {
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
// ListPack returns the list of blobs saved in the pack id and the length of
// the pack header.
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, hdrSize uint32, err error)
// ListPack returns the list of blobs saved in the pack id.
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error
+17
View File
@@ -1,7 +1,9 @@
package restic
import (
"crypto/rand"
"fmt"
"io"
)
// TestParseID parses s as a ID and panics if that fails.
@@ -18,3 +20,18 @@ func TestParseID(s string) ID {
func TestParseHandle(s string, t BlobType) BlobHandle {
return BlobHandle{ID: TestParseID(s), Type: t}
}
func NewRandomBlobHandle() BlobHandle {
return BlobHandle{ID: NewRandomID(), Type: DataBlob}
}
// NewRandomID returns a randomly generated ID. When reading from rand fails,
// the function panics.
func NewRandomID() ID {
id := ID{}
_, err := io.ReadFull(rand.Reader, id[:])
if err != nil {
panic(err)
}
return id
}
-3
View File
@@ -80,12 +80,9 @@ func extractToFile(buf []byte, filename, target string, printf func(string, ...i
return err
}
if err = newFile.Sync(); err != nil {
_ = newFile.Close()
_ = os.Remove(newFile.Name())
return err
}
if err = newFile.Close(); err != nil {
_ = os.Remove(newFile.Name())
return err
}
-2
View File
@@ -87,7 +87,6 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
var msg githubError
jerr := json.NewDecoder(res.Body).Decode(&msg)
if jerr == nil {
_ = res.Body.Close()
return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n %v", res.StatusCode, res.Status, msg.Message)
}
}
@@ -138,7 +137,6 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
}
if res.StatusCode != http.StatusOK {
_ = res.Body.Close()
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
+11 -12
View File
@@ -137,18 +137,10 @@ func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
// Env creates a test environment and extracts the repository fixture.
// Returned is the repo path and a cleanup function.
func Env(t testing.TB, repoFixture string) string {
func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
t.Helper()
var tempdir string
if TestCleanupTempDirs {
tempdir = t.TempDir()
} else {
var err error
tempdir, err = os.MkdirTemp(TestTempDir, "restic-test-env-")
OK(t, err)
t.Logf("leaving temporary directory %v used for test", tempdir)
}
tempdir, err := os.MkdirTemp(TestTempDir, "restic-test-env-")
OK(t, err)
fd, err := os.Open(repoFixture)
if err != nil {
@@ -158,7 +150,14 @@ func Env(t testing.TB, repoFixture string) string {
SetupTarTestFixture(t, tempdir, repoFixture)
return filepath.Join(tempdir, "repo")
return filepath.Join(tempdir, "repo"), func() {
if !TestCleanupTempDirs {
t.Logf("leaving temporary directory %v used for test", tempdir)
return
}
RemoveAll(t, tempdir)
}
}
func isFile(fi os.FileInfo) bool {