amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-06-17 22:24:17 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #21841 from MichaelEischer/isolate-repository

Browse Source 
prevent imports of repository internals
This commit is contained in:
Michael Eischer
2026-06-10 22:19:24 +02:00
committed by GitHub
parent 81571775d5 7d25ca9d67
commit 17ff3aa5f9
16 changed files with 543 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.golangci.yml
+7
View File
@@ -40,6 +40,13 @@ linters:
desc: "internal/restic should not be imported to keep the architectural layers intact"
- pkg: "github.com/restic/restic/internal/repository"
desc: "internal/repository should not be imported to keep the architectural layers intact"
repository-internals:
files:
- "**"
- "!**/internal/repository/**"
deny:
- pkg: "github.com/restic/restic/internal/repository/"
desc: "packages below internal/repository should not be imported to not depend on repository internals"
importas:
alias:
- pkg: github.com/restic/restic/internal/test
cmd/restic/cmd_copy_integration_test.go
+1 -1
View File
@@ -103,7 +103,7 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
defer unlock()
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(context.TODO(), id, size)
blobs, err := repo.ListPack(context.TODO(), id, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
cmd/restic/cmd_debug.go
+10 -319
View File
@@ -4,28 +4,18 @@ package main
import (
"context"
"crypto/aes"
"crypto/cipher"
"encoding/json"
"fmt"
"io"
"os"
"runtime"
"sync"
"time"
"github.com/klauspost/compress/zstd"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
@@ -147,7 +137,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer,
var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size)
blobs, err := repo.ListPack(ctx, id, size)
if err != nil {
printer.E("error for pack %v: %v", id.Str(), err)
return nil
@@ -172,17 +162,6 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer,
})
}
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id)
if err != nil {
return err
}
return idx.Dump(wr)
})
}
func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
@@ -200,7 +179,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
switch tpe {
case "indexes":
return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
case "snapshots":
return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
case "packs":
@@ -213,7 +192,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
}
printer.S("indexes:")
err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
if err != nil {
return err
}
@@ -224,225 +203,6 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
}
}
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
if bytewise {
printer.S(" trying to repair blob by finding a broken byte")
} else {
printer.S(" trying to repair blob with single bit flip")
}
ch := make(chan int)
var wg errgroup.Group
done := make(chan struct{})
var fixed []byte
var found bool
workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ {
wg.Go(func() error {
// make a local copy of the buffer
buf := make([]byte, len(input))
copy(buf, input)
testFlip := func(idx int, pattern byte) bool {
// flip bits
buf[idx] ^= pattern
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil {
printer.S("")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext))
close(done)
found = true
fixed = plaintext
return true
}
// flip bits back
buf[idx] ^= pattern
return false
}
for i := range ch {
if bytewise {
for j := 0; j < 255; j++ {
if testFlip(i, byte(j)) {
return nil
}
}
} else {
for j := 0; j < 7; j++ {
// flip each bit once
if testFlip(i, (1 << uint(j))) {
return nil
}
}
}
}
return nil
})
}
wg.Go(func() error {
defer close(ch)
start := time.Now()
info := time.Now()
for i := range input {
select {
case ch <- i:
case <-done:
printer.S(" done after %v", time.Since(start))
return nil
}
if time.Since(info) > time.Second {
secs := time.Since(start).Seconds()
gps := float64(i) / secs
remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now()
}
}
return nil
})
err := wg.Wait()
if err != nil {
panic("all go routines can only return nil")
}
if !found {
printer.S("\n blob could not be repaired")
}
return fixed
}
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
// strip signature at the end
l := len(buf)
nonce, ct := buf[:16], buf[16:l-16]
out := make([]byte, len(ct))
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCTR(c, nonce)
e.XORKeyStream(out, ct)
return out
}
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
// allow processing broken pack files
if pack == nil {
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(pack) {
printer.E("skipping truncated blob")
continue
}
buf := pack[blob.Offset : blob.Offset+blob.Length]
key := repo.Key()
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
outputPrefix := ""
filePrefix := ""
if err != nil {
printer.E("error decrypting blob: %v", err)
if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
if err != nil {
return err
}
continue
}
}
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
printer.S(" failed to decompress blob %v", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
}
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
if err != nil {
return err
}
}
if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
printer.S(" uploaded %v %v", blob.Type, id)
}
}
return nil
})
return err
}
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename)
if err != nil {
return err
}
_, err = f.Write(plain)
if err != nil {
_ = f.Close()
return err
}
err = f.Close()
if err != nil {
return err
}
printer.S("decrypt of blob %v stored at %v", id, filename)
return nil
}
func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
@@ -478,8 +238,14 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
return err
}
examineOpts := repository.ExaminePackOptions{
TryRepair: opts.TryRepair,
RepairByte: opts.RepairByte,
ExtractPack: opts.ExtractPack,
ReuploadBlobs: opts.ReuploadBlobs,
}
for _, id := range ids {
err := examinePack(ctx, opts, repo, id, printer)
err := repository.ExaminePack(ctx, repo, id, examineOpts, printer)
if err != nil {
printer.E("error: %v", err)
}
@@ -489,78 +255,3 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
}
return nil
}
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
printer.S("examine %v", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
printer.S(" file size is %v", len(buf))
gotID := restic.Hash(buf)
if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID)
} else {
printer.S(" hash for file content matches")
}
printer.S(" ========================================")
printer.S(" looking for info in the indexes")
blobsLoaded := false
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
continue
}
checkPackSize(blobs, len(buf), printer)
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
if err != nil {
printer.E("error: %v", err)
} else {
blobsLoaded = true
}
}
printer.S(" ========================================")
printer.S(" inspect the pack itself")
blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
checkPackSize(blobs, len(buf), printer)
if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer)
}
return nil
}
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
// track current size and offset
var size, offset uint64
blobs.Sort()
for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
}
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
}
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
} else {
printer.S(" file sizes match")
}
}
cmd/restic/cmd_find.go
+1 -1
View File
@@ -472,7 +472,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
delete(packIDs, idStr)
}
debug.Log("Found pack %s", idStr)
blobs, _, err := f.repo.ListPack(ctx, id, size)
blobs, err := f.repo.ListPack(ctx, id, size)
if err != nil {
return err
}
cmd/restic/cmd_list.go
+7 -12
View File
@@ -6,7 +6,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
@@ -69,18 +69,13 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T
case "locks":
t = restic.LockFile
case "blobs":
return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
if err != nil {
return err
for entry := range repository.AllIndexBlobs(ctx, repo, repo) {
if entry.Error != nil {
return entry.Error
}
for blobs := range idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
}
printer.S("%v %v", blobs.Type, blobs.ID)
}
return nil
})
printer.S("%v %v", entry.Handle.Type, entry.Handle.ID)
}
return nil
default:
return errors.Fatal("invalid type")
}
cmd/restic/cmd_repair_index_integration_test.go
-10
View File
@@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -61,15 +60,6 @@ func TestRebuildIndex(t *testing.T) {
testRebuildIndex(t, nil)
}
func TestRebuildIndexAlwaysFull(t *testing.T) {
indexFull := index.Full
defer func() {
index.Full = indexFull
}()
index.Full = func(*index.Index) bool { return true }
testRebuildIndex(t, nil)
}
// indexErrorBackend modifies the first index after reading.
type indexErrorBackend struct {
backend.Backend
internal/checker/checker_test.go
+6 -45
View File
@@ -4,7 +4,6 @@ import (
"context"
"io"
"math/rand"
"os"
"path/filepath"
"sort"
"strconv"
@@ -19,7 +18,6 @@ import (
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/hashing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@@ -192,56 +190,19 @@ func TestModifiedIndex(t *testing.T) {
Type: restic.IndexFile,
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
tmpfile, err := os.CreateTemp("", "restic-test-mod-index-")
if err != nil {
t.Fatal(err)
}
defer func() {
err := tmpfile.Close()
if err != nil {
t.Fatal(err)
}
err = os.Remove(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
}()
wr := io.Writer(tmpfile)
var hw *hashing.Writer
if be.Hasher() != nil {
hw = hashing.NewWriter(wr, be.Hasher())
wr = hw
}
// read the file from the backend
err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
_, err := io.Copy(wr, rd)
var data []byte
test.OK(t, be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
var err error
data, err = io.ReadAll(rd)
return err
})
test.OK(t, err)
}))
// save the index again with a modified name so that the hash doesn't match
// the content any more
h2 := backend.Handle{
Type: restic.IndexFile,
Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
var hash []byte
if hw != nil {
hash = hw.Sum(nil)
}
rd, err := backend.NewFileReader(tmpfile, hash)
if err != nil {
t.Fatal(err)
}
err = be.Save(context.TODO(), h2, rd)
if err != nil {
t.Fatal(err)
}
test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher())))
chkr := checker.New(repo, false)
hints, errs := chkr.LoadIndex(context.TODO(), nil)
internal/repository/debug.go
+338
View File
@@ -0,0 +1,338 @@
//go:build debug
package repository
import (
"context"
"crypto/aes"
"crypto/cipher"
"fmt"
"io"
"os"
"runtime"
"time"
"github.com/klauspost/compress/zstd"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
)
// DumpIndexes loads each on-disk index file and writes its debug dump to wr.
func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id)
if err != nil {
return err
}
return idx.Dump(wr)
})
}
// ExaminePackOptions configures debug examination of a pack file.
type ExaminePackOptions struct {
TryRepair bool
RepairByte bool
ExtractPack bool
ReuploadBlobs bool
}
// ExaminePack loads and inspects a pack file and its index entries.
func ExaminePack(ctx context.Context, repo restic.Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error {
printer.S("examine %v", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
printer.S(" file size is %v", len(buf))
gotID := restic.Hash(buf)
if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID)
} else {
printer.S(" hash for file content matches")
}
printer.S(" ========================================")
printer.S(" looking for info in the indexes")
blobsLoaded := false
// examine all data the indexes have for the pack file
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
blobs := b.Blobs
if len(blobs) == 0 {
continue
}
checkPackSize(blobs, len(buf), printer)
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
if err != nil {
printer.E("error: %v", err)
} else {
blobsLoaded = true
}
}
printer.S(" ========================================")
printer.S(" inspect the pack itself")
blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
checkPackSize(blobs, len(buf), printer)
if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer)
}
return nil
}
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
// track current size and offset
var size, offset uint64
blobs.Sort()
for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
}
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
}
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
} else {
printer.S(" file sizes match")
}
}
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
if bytewise {
printer.S(" trying to repair blob by finding a broken byte")
} else {
printer.S(" trying to repair blob with single bit flip")
}
ch := make(chan int)
var wg errgroup.Group
done := make(chan struct{})
var fixed []byte
var found bool
workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ {
wg.Go(func() error {
// make a local copy of the buffer
buf := make([]byte, len(input))
copy(buf, input)
testFlip := func(idx int, pattern byte) bool {
// flip bits
buf[idx] ^= pattern
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil {
printer.S("")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext))
close(done)
found = true
fixed = plaintext
return true
}
// flip bits back
buf[idx] ^= pattern
return false
}
for i := range ch {
if bytewise {
for j := 0; j < 255; j++ {
if testFlip(i, byte(j)) {
return nil
}
}
} else {
for j := 0; j < 7; j++ {
// flip each bit once
if testFlip(i, (1 << uint(j))) {
return nil
}
}
}
}
return nil
})
}
wg.Go(func() error {
defer close(ch)
start := time.Now()
info := time.Now()
for i := range input {
select {
case ch <- i:
case <-done:
printer.S(" done after %v", time.Since(start))
return nil
}
if time.Since(info) > time.Second {
secs := time.Since(start).Seconds()
gps := float64(i) / secs
remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now()
}
}
return nil
})
err := wg.Wait()
if err != nil {
panic("all go routines can only return nil")
}
if !found {
printer.S("\n blob could not be repaired")
}
return fixed
}
func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
// strip signature at the end
l := len(buf)
nonce, ct := buf[:16], buf[16:l-16]
out := make([]byte, len(ct))
c, err := aes.NewCipher(k.EncryptionKey[:])
if err != nil {
panic(fmt.Sprintf("unable to create cipher: %v", err))
}
e := cipher.NewCTR(c, nonce)
e.XORKeyStream(out, ct)
return out
}
func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
packData, err := repo.LoadRaw(ctx, restic.PackFile, packID)
// allow processing broken pack files
if packData == nil {
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(packData) {
printer.E("skipping truncated blob")
continue
}
buf := packData[blob.Offset : blob.Offset+blob.Length]
key := repo.Key()
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
outputPrefix := ""
filePrefix := ""
if err != nil {
printer.E("error decrypting blob: %v", err)
if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
if err != nil {
return err
}
continue
}
}
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
printer.S(" failed to decompress blob %v", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
}
}
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
if err != nil {
return err
}
}
if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
printer.S(" uploaded %v %v", blob.Type, id)
}
}
return nil
})
return err
}
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename)
if err != nil {
return err
}
_, err = f.Write(plain)
if err != nil {
_ = f.Close()
return err
}
err = f.Close()
if err != nil {
return err
}
printer.S("decrypt of blob %v stored at %v", id, filename)
return nil
}
internal/repository/index/master_index_test.go
+59
View File
@@ -694,3 +694,62 @@ func TestRewriteSplitPacks(t *testing.T) {
blobs := mi.Lookup(blobOther.BlobHandle)
rtest.Equals(t, nil, blobs)
}
// TestRewriteFullPacks checks that Rewrite drops a duplicate full index for the same
// pack while keeping the other index files and blob lookups intact. Creates 3 indexes:
// - indexA: contains packA
// - indexB: contains packB
// - indexC: contains packB
// After the rewrite, indexC must be dropped. The other indexes must be kept.
func TestRewriteFullPacks(t *testing.T) {
originalFull := index.Full
defer func() {
index.Full = originalFull
}()
index.Full = func(*index.Index) bool { return true }
repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion)
packA := restic.NewRandomID()
packB := restic.NewRandomID()
blobA := restic.PackedBlob{
PackID: packA,
Blob: restic.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(10)),
Offset: 0,
},
}
blobB := restic.PackedBlob{
PackID: packB,
Blob: restic.Blob{
BlobHandle: restic.NewRandomBlobHandle(),
Length: uint(crypto.CiphertextLength(50)),
Offset: 0,
},
}
mi := index.NewMasterIndex()
rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
rtest.OK(t, mi.Flush(context.TODO(), unpacked))
indexIDs := mi.IDs()
rtest.Equals(t, 3, len(indexIDs))
rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
mi2 := index.NewMasterIndex()
rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
afterRewrite := mi2.IDs()
rtest.Equals(t, 2, len(afterRewrite))
rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
}
internal/repository/index_list.go
+41
View File
@@ -0,0 +1,41 @@
package repository
import (
"context"
"iter"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
)
// IndexBlob is one blob handle from an on-disk index file, or an error from loading/decoding
// that file.
type IndexBlob struct {
Handle restic.BlobHandle
Error error
}
// AllIndexBlobs streams blob handles from each index file without building a master index.
func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.LoaderUnpacked) iter.Seq[IndexBlob] {
return func(yield func(IndexBlob) bool) {
stopIteration := errors.New("stop index blob iteration")
err := index.ForAllIndexes(ctx, lister, loader, func(_ restic.ID, idx *index.Index, err error) error {
if err != nil {
return err
}
for blob := range idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
}
if !yield(IndexBlob{Handle: blob.BlobHandle}) {
return stopIteration
}
}
return nil
})
if err != nil && !errors.Is(err, stopIteration) {
yield(IndexBlob{Error: err})
}
}
}
internal/repository/index_list_test.go
+62
View File
@@ -0,0 +1,62 @@
package repository_test
import (
"context"
"testing"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestAllIndexBlobs(t *testing.T) {
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
want := restic.NewBlobSet()
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for i := range 5 {
data := []byte{byte('a' + i)}
id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, data, restic.ID{}, false)
rtest.OK(t, err)
want.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id})
}
return nil
}))
rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
fromMaster := restic.NewBlobSet()
rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
fromMaster.Insert(pb.BlobHandle)
}))
rtest.Equals(t, want, fromMaster)
fromStream := restic.NewBlobSet()
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
if entry.Error != nil {
t.Fatalf("unexpected error: %v", entry.Error)
}
fromStream.Insert(entry.Handle)
}
rtest.Equals(t, want, fromStream)
}
func TestAllIndexBlobsEarlyStop(t *testing.T) {
repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for range 5 {
_, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, []byte("test"), restic.ID{}, false)
rtest.OK(t, err)
}
return nil
}))
var count int
for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
rtest.Assert(t, entry.Error == nil, "unexpected error after early stop: %v", entry.Error)
count++
break
}
rtest.Equals(t, 1, count)
}
internal/repository/repack_test.go
+1 -1
View File
@@ -86,7 +86,7 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
blobs := restic.NewBlobSet()
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
entries, _, err := repo.ListPack(context.TODO(), id, size)
entries, err := repo.ListPack(context.TODO(), id, size)
if err != nil {
t.Fatalf("error listing pack %v: %v", id, err)
}
internal/repository/repair_pack.go
+1 -1
View File
@@ -76,7 +76,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
if ids.Has(id) {
blobs, _, err := repo.ListPack(ctx, id, size)
blobs, err := repo.ListPack(ctx, id, size)
if err != nil {
return nil
}
internal/repository/repository.go
+6 -7
View File
@@ -781,7 +781,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
worker := func() error {
for fi := range ch {
entries, _, err := r.ListPack(wgCtx, fi.ID, fi.Size)
entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
if err != nil {
debug.Log("unable to list pack file %v", fi.ID.Str())
m.Lock()
@@ -957,12 +957,11 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
})
}
// ListPack returns the list of blobs saved in the pack id and the length of
// the pack header.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, uint32, error) {
// ListPack returns the list of blobs saved in the pack id.
func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
if err != nil {
if r.cache != nil {
// ignore error as there is not much we can do here
@@ -970,9 +969,9 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
}
// retry on error
entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
}
return entries, hdrSize, err
return entries, err
}
// Delete calls backend.Delete() if implemented, and returns an error
internal/repository/repository_test.go
+1 -1
View File
@@ -479,7 +479,7 @@ func TestListPack(t *testing.T) {
return nil
}))
blobs, _, err := repo.ListPack(context.TODO(), packID, size)
blobs, err := repo.ListPack(context.TODO(), packID, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
internal/restic/repository.go
+2 -3
View File
@@ -32,9 +32,8 @@ type Repository interface {
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
ListBlobs(ctx context.Context, fn func(PackedBlob)) error
ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
// ListPack returns the list of blobs saved in the pack id and the length of
// the pack header.
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, hdrSize uint32, err error)
// ListPack returns the list of blobs saved in the pack id.
ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error