Merge pull request #21868 from MichaelEischer/fix-zero-sized-blobs

repository: fix zero-sized blobs in v2 repos

.golangci.yml

@@ -32,7 +32,6 @@ linters:
         backend-imports:
           files:
             - "**/internal/backend/**"
-            - "!**/internal/backend/cache/**"
             - "!**/internal/backend/test/**"
             - "!**/*_test.go"
           deny:
@@ -40,6 +39,13 @@ linters:
               desc: "internal/restic should not be imported to keep the architectural layers intact"
             - pkg: "github.com/restic/restic/internal/repository"
               desc: "internal/repository should not be imported to keep the architectural layers intact"
+        repository-internals:
+          files:
+            - "**"
+            - "!**/internal/repository/**"
+          deny:
+            - pkg: "github.com/restic/restic/internal/repository/"
+              desc: "packages below internal/repository should not be imported to not depend on repository internals"
     importas:
       alias:
         - pkg: github.com/restic/restic/internal/test

VERSION

@@ -1 +1 @@
-0.19.0
+0.19.0-dev

cmd/restic/cmd_check.go

@@ -307,7 +307,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
 	go chkr.Packs(ctx, errChan)
 
 	for err := range errChan {
-		var packErr *repository.PackError
+		var packErr *repository.ErrPackMetadata
 		if errors.As(err, &packErr) {
 			if packErr.Orphaned {
 				orphanedPacks++

cmd/restic/cmd_copy_integration_test.go

@@ -103,7 +103,7 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
 		defer unlock()
 
 		rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
-			blobs, _, err := repo.ListPack(context.TODO(), id, size)
+			blobs, err := repo.ListPack(context.TODO(), id, size)
 			rtest.OK(t, err)
 			rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
 

cmd/restic/cmd_debug.go

@@ -4,31 +4,19 @@ package main
 
 import (
 	"context"
-	"crypto/aes"
-	"crypto/cipher"
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
-	"runtime"
-	"sync"
-	"time"
 
-	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-	"golang.org/x/sync/errgroup"
 
-	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/repository/index"
-	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
-	"github.com/restic/restic/internal/ui/progress"
 )
 
 func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) {
@@ -128,61 +116,6 @@ func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io
 	})
 }
 
-// Pack is the struct used in printPacks.
-type Pack struct {
-	Name string `json:"name"`
-
-	Blobs []Blob `json:"blobs"`
-}
-
-// Blob is the struct used in printPacks.
-type Blob struct {
-	Type   restic.BlobType `json:"type"`
-	Length uint            `json:"length"`
-	ID     restic.ID       `json:"id"`
-	Offset uint            `json:"offset"`
-}
-
-func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error {
-
-	var m sync.Mutex
-	return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
-		blobs, _, err := repo.ListPack(ctx, id, size)
-		if err != nil {
-			printer.E("error for pack %v: %v", id.Str(), err)
-			return nil
-		}
-
-		p := Pack{
-			Name:  id.String(),
-			Blobs: make([]Blob, len(blobs)),
-		}
-		for i, blob := range blobs {
-			p.Blobs[i] = Blob{
-				Type:   blob.Type,
-				Length: blob.Length,
-				ID:     blob.ID,
-				Offset: blob.Offset,
-			}
-		}
-
-		m.Lock()
-		defer m.Unlock()
-		return prettyPrintJSON(wr, p)
-	})
-}
-
-func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
-	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
-		printer.S("index_id: %v", id)
-		if err != nil {
-			return err
-		}
-
-		return idx.Dump(wr)
-	})
-}
-
 func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
 	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
 
@@ -200,11 +133,11 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
 
 	switch tpe {
 	case "indexes":
-		return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
+		return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
 	case "snapshots":
 		return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
 	case "packs":
-		return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
+		return repository.DumpPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
 	case "all":
 		printer.S("snapshots:")
 		err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
@@ -213,7 +146,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
 		}
 
 		printer.S("indexes:")
-		err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
+		err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
 		if err != nil {
 			return err
 		}
@@ -224,225 +157,6 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
 	}
 }
 
-func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
-	if bytewise {
-		printer.S("        trying to repair blob by finding a broken byte")
-	} else {
-		printer.S("        trying to repair blob with single bit flip")
-	}
-
-	ch := make(chan int)
-	var wg errgroup.Group
-	done := make(chan struct{})
-	var fixed []byte
-	var found bool
-
-	workers := runtime.GOMAXPROCS(0)
-	printer.S("         spinning up %d worker functions", runtime.GOMAXPROCS(0))
-	for i := 0; i < workers; i++ {
-		wg.Go(func() error {
-			// make a local copy of the buffer
-			buf := make([]byte, len(input))
-			copy(buf, input)
-
-			testFlip := func(idx int, pattern byte) bool {
-				// flip bits
-				buf[idx] ^= pattern
-
-				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
-				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
-				if err == nil {
-					printer.S("")
-					printer.S("        blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
-					printer.S("        hash is %v", restic.Hash(plaintext))
-					close(done)
-					found = true
-					fixed = plaintext
-					return true
-				}
-
-				// flip bits back
-				buf[idx] ^= pattern
-				return false
-			}
-
-			for i := range ch {
-				if bytewise {
-					for j := 0; j < 255; j++ {
-						if testFlip(i, byte(j)) {
-							return nil
-						}
-					}
-				} else {
-					for j := 0; j < 7; j++ {
-						// flip each bit once
-						if testFlip(i, (1 << uint(j))) {
-							return nil
-						}
-					}
-				}
-			}
-			return nil
-		})
-	}
-
-	wg.Go(func() error {
-		defer close(ch)
-
-		start := time.Now()
-		info := time.Now()
-		for i := range input {
-			select {
-			case ch <- i:
-			case <-done:
-				printer.S("     done after %v", time.Since(start))
-				return nil
-			}
-
-			if time.Since(info) > time.Second {
-				secs := time.Since(start).Seconds()
-				gps := float64(i) / secs
-				remaining := len(input) - i
-				eta := time.Duration(float64(remaining)/gps) * time.Second
-
-				printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
-					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
-				info = time.Now()
-			}
-		}
-		return nil
-	})
-	err := wg.Wait()
-	if err != nil {
-		panic("all go routines can only return nil")
-	}
-
-	if !found {
-		printer.S("\n        blob could not be repaired")
-	}
-	return fixed
-}
-
-func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
-	// strip signature at the end
-	l := len(buf)
-	nonce, ct := buf[:16], buf[16:l-16]
-	out := make([]byte, len(ct))
-
-	c, err := aes.NewCipher(k.EncryptionKey[:])
-	if err != nil {
-		panic(fmt.Sprintf("unable to create cipher: %v", err))
-	}
-	e := cipher.NewCTR(c, nonce)
-	e.XORKeyStream(out, ct)
-
-	return out
-}
-
-func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
-	dec, err := zstd.NewReader(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
-	// allow processing broken pack files
-	if pack == nil {
-		return err
-	}
-
-	err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		for _, blob := range list {
-			printer.S("      loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
-			if int(blob.Offset+blob.Length) > len(pack) {
-				printer.E("skipping truncated blob")
-				continue
-			}
-			buf := pack[blob.Offset : blob.Offset+blob.Length]
-			key := repo.Key()
-
-			nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
-			plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
-			outputPrefix := ""
-			filePrefix := ""
-			if err != nil {
-				printer.E("error decrypting blob: %v", err)
-				if opts.TryRepair || opts.RepairByte {
-					plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
-				}
-				if plaintext != nil {
-					outputPrefix = "repaired "
-					filePrefix = "repaired-"
-				} else {
-					plaintext = decryptUnsigned(key, buf)
-					err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
-					if err != nil {
-						return err
-					}
-					continue
-				}
-			}
-
-			if blob.IsCompressed() {
-				decompressed, err := dec.DecodeAll(plaintext, nil)
-				if err != nil {
-					printer.S("         failed to decompress blob %v", blob.ID)
-				}
-				if decompressed != nil {
-					plaintext = decompressed
-				}
-			}
-
-			id := restic.Hash(plaintext)
-			var prefix string
-			if !id.Equal(blob.ID) {
-				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
-				prefix = "wrong-hash-"
-			} else {
-				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
-				prefix = "correct-"
-			}
-			if opts.ExtractPack {
-				err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
-				if err != nil {
-					return err
-				}
-			}
-			if opts.ReuploadBlobs {
-				_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
-				if err != nil {
-					return err
-				}
-				printer.S("         uploaded %v %v", blob.Type, id)
-			}
-		}
-		return nil
-	})
-	return err
-}
-
-func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
-	filename := fmt.Sprintf("%s%s.bin", prefix, id)
-	f, err := os.Create(filename)
-	if err != nil {
-		return err
-	}
-
-	_, err = f.Write(plain)
-	if err != nil {
-		_ = f.Close()
-		return err
-	}
-
-	err = f.Close()
-	if err != nil {
-		return err
-	}
-
-	printer.S("decrypt of blob %v stored at %v", id, filename)
-	return nil
-}
-
 func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
 	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
 
@@ -478,8 +192,14 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
 		return err
 	}
 
+	examineOpts := repository.ExaminePackOptions{
+		TryRepair:     opts.TryRepair,
+		RepairByte:    opts.RepairByte,
+		ExtractPack:   opts.ExtractPack,
+		ReuploadBlobs: opts.ReuploadBlobs,
+	}
 	for _, id := range ids {
-		err := examinePack(ctx, opts, repo, id, printer)
+		err := repository.ExaminePack(ctx, repo, id, examineOpts, printer)
 		if err != nil {
 			printer.E("error: %v", err)
 		}
@@ -489,78 +209,3 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
 	}
 	return nil
 }
-
-func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
-	printer.S("examine %v", id)
-
-	buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
-	// also process damaged pack files
-	if buf == nil {
-		return err
-	}
-	printer.S("  file size is %v", len(buf))
-	gotID := restic.Hash(buf)
-	if !id.Equal(gotID) {
-		printer.S("  wanted hash %v, got %v", id, gotID)
-	} else {
-		printer.S("  hash for file content matches")
-	}
-
-	printer.S("  ========================================")
-	printer.S("  looking for info in the indexes")
-
-	blobsLoaded := false
-	// examine all data the indexes have for the pack file
-	for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
-		blobs := b.Blobs
-		if len(blobs) == 0 {
-			continue
-		}
-
-		checkPackSize(blobs, len(buf), printer)
-
-		err = loadBlobs(ctx, opts, repo, id, blobs, printer)
-		if err != nil {
-			printer.E("error: %v", err)
-		} else {
-			blobsLoaded = true
-		}
-	}
-
-	printer.S("  ========================================")
-	printer.S("  inspect the pack itself")
-
-	blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
-	if err != nil {
-		return fmt.Errorf("pack %v: %v", id.Str(), err)
-	}
-	checkPackSize(blobs, len(buf), printer)
-
-	if !blobsLoaded {
-		return loadBlobs(ctx, opts, repo, id, blobs, printer)
-	}
-	return nil
-}
-
-func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
-	// track current size and offset
-	var size, offset uint64
-
-	blobs.Sort()
-
-	for _, pb := range blobs {
-		printer.S("      %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
-		if offset != uint64(pb.Offset) {
-			printer.S("      hole in file, want offset %v, got %v", offset, pb.Offset)
-		}
-		offset = uint64(pb.Offset + pb.Length)
-		size += uint64(pb.Length)
-	}
-	size += uint64(pack.CalculateHeaderSize(blobs))
-
-	if uint64(fileSize) != size {
-		printer.S("      file sizes do not match: computed %v, file size is %v", size, fileSize)
-	} else {
-		printer.S("      file sizes match")
-	}
-}

cmd/restic/cmd_find.go

@@ -472,7 +472,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 			delete(packIDs, idStr)
 		}
 		debug.Log("Found pack %s", idStr)
-		blobs, _, err := f.repo.ListPack(ctx, id, size)
+		blobs, err := f.repo.ListPack(ctx, id, size)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_list.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
-	"github.com/restic/restic/internal/repository/index"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
 
@@ -69,18 +69,13 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
-			if err != nil {
-				return err
+		for entry := range repository.AllIndexBlobs(ctx, repo, repo) {
+			if entry.Error != nil {
+				return entry.Error
 			}
-			for blobs := range idx.Values() {
-				if ctx.Err() != nil {
-					return ctx.Err()
-				}
-				printer.S("%v %v", blobs.Type, blobs.ID)
-			}
-			return nil
-		})
+			printer.S("%v %v", entry.Handle.Type, entry.Handle.ID)
+		}
+		return nil
 	default:
 		return errors.Fatal("invalid type")
 	}

cmd/restic/cmd_repair_index_integration_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
-	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -61,15 +60,6 @@ func TestRebuildIndex(t *testing.T) {
 	testRebuildIndex(t, nil)
 }
 
-func TestRebuildIndexAlwaysFull(t *testing.T) {
-	indexFull := index.Full
-	defer func() {
-		index.Full = indexFull
-	}()
-	index.Full = func(*index.Index) bool { return true }
-	testRebuildIndex(t, nil)
-}
-
 // indexErrorBackend modifies the first index after reading.
 type indexErrorBackend struct {
 	backend.Backend

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/go-ole/go-ole v1.3.0
 	github.com/google/go-cmp v0.7.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/klauspost/compress v1.18.4
+	github.com/klauspost/compress v1.18.6
 	github.com/minio/minio-go/v7 v7.1.0
 	github.com/ncw/swift/v2 v2.0.5
 	github.com/peterbourgon/unixtransport v0.0.7

go.sum

@@ -131,8 +131,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
 github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
+github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
 github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=

helpers/prepare-release/main.go

@@ -333,7 +333,7 @@ func updateVersionDev() {
 		die("unable to write version to file: %v", err)
 	}
 
-	newVersion := fmt.Sprintf(`var version = "%s-dev (compiled manually)"`, opts.Version)
+	newVersion := fmt.Sprintf(`const Version = "%s-dev (compiled manually)"`, opts.Version)
 	replace(versionCodeFile, versionPattern, newVersion)
 
 	msg("committing cmd/restic/global.go with dev version")

internal/backend/azure/azure.go

@@ -31,11 +31,9 @@ import (
 
 // Backend stores data on an azure endpoint.
 type Backend struct {
-	cfg          Config
-	container    *azContainer.Client
-	connections  uint
-	prefix       string
-	listMaxItems int
+	cfg         Config
+	container   *azContainer.Client
+	connections uint
 	layout.Layout
 
 	accessTier blob.AccessTier
@@ -145,12 +143,11 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	}
 
 	be := &Backend{
-		container:    client,
-		cfg:          cfg,
-		connections:  cfg.Connections,
-		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
-		listMaxItems: defaultListMaxItems,
-		accessTier:   accessTier,
+		container:   client,
+		cfg:         cfg,
+		connections: cfg.Connections,
+		Layout:      layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		accessTier:  accessTier,
 	}
 
 	return be, nil
@@ -195,11 +192,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
 	return be, nil
 }
 
-// SetListMaxItems sets the number of list items to load per request.
-func (be *Backend) SetListMaxItems(i int) {
-	be.listMaxItems = i
-}
-
 // IsNotExist returns true if the error is caused by a not existing file.
 func (be *Backend) IsNotExist(err error) bool {
 	return bloberror.HasCode(err, bloberror.BlobNotFound)
@@ -231,11 +223,6 @@ func (be *Backend) Hasher() hash.Hash {
 	return md5.New()
 }
 
-// Path returns the path in the bucket that is used for this backend.
-func (be *Backend) Path() string {
-	return be.prefix
-}
-
 // useAccessTier determines whether to apply the configured access tier to a given file.
 // For archive access tier, only data files are stored using that class; metadata
 // must remain instantly accessible.
@@ -419,7 +406,7 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
 		prefix += "/"
 	}
 
-	maxI := int32(be.listMaxItems)
+	maxI := int32(defaultListMaxItems)
 
 	opts := &azContainer.ListBlobsFlatOptions{
 		MaxResults: &maxI,

internal/backend/b2/b2.go

@@ -23,10 +23,9 @@ import (
 
 // b2Backend is a backend which stores its data on Backblaze B2.
 type b2Backend struct {
-	client       *b2.Client
-	bucket       *b2.Bucket
-	cfg          Config
-	listMaxItems int
+	client *b2.Client
+	bucket *b2.Bucket
+	cfg    Config
 	layout.Layout
 
 	canDelete bool
@@ -107,12 +106,11 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string,
 	}
 
 	be := &b2Backend{
-		client:       client,
-		bucket:       bucket,
-		cfg:          cfg,
-		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
-		listMaxItems: defaultListMaxItems,
-		canDelete:    true,
+		client:    client,
+		bucket:    bucket,
+		cfg:       cfg,
+		Layout:    layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		canDelete: true,
 	}
 
 	return be, nil
@@ -140,20 +138,14 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
 	}
 
 	be := &b2Backend{
-		client:       client,
-		bucket:       bucket,
-		cfg:          cfg,
-		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
-		listMaxItems: defaultListMaxItems,
+		client: client,
+		bucket: bucket,
+		cfg:    cfg,
+		Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
 	}
 	return be, nil
 }
 
-// SetListMaxItems sets the number of list items to load per request.
-func (be *b2Backend) SetListMaxItems(i int) {
-	be.listMaxItems = i
-}
-
 func (be *b2Backend) Properties() backend.Properties {
 	return backend.Properties{
 		Connections:      be.cfg.Connections,
@@ -304,7 +296,7 @@ func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backe
 	defer cancel()
 
 	prefix, _ := be.Basedir(t)
-	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems))
+	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(defaultListMaxItems))
 
 	for iter.Next() {
 		obj := iter.Object()

internal/backend/cache/backend.go

@@ -7,11 +7,10 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/restic"
 )
 
-// Backend wraps a restic.Backend and adds a cache.
-type Backend struct {
+// cacheBackend wraps a restic.cacheBackend and adds a cache.
+type cacheBackend struct {
 	backend.Backend
 	*Cache
 
@@ -24,10 +23,10 @@ type Backend struct {
 }
 
 // ensure Backend implements backend.Backend
-var _ backend.Backend = &Backend{}
+var _ backend.Backend = &cacheBackend{}
 
-func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *Backend {
-	return &Backend{
+func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *cacheBackend {
+	return &cacheBackend{
 		Backend:    be,
 		Cache:      c,
 		inProgress: make(map[backend.Handle]chan struct{}),
@@ -36,7 +35,7 @@ func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface
 }
 
 // Remove deletes a file from the backend and the cache if it has been cached.
-func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
+func (b *cacheBackend) Remove(ctx context.Context, h backend.Handle) error {
 	debug.Log("cache Remove(%v)", h)
 	err := b.Backend.Remove(ctx, h)
 	if err != nil {
@@ -58,7 +57,7 @@ func autoCacheTypes(h backend.Handle) bool {
 }
 
 // Save stores a new file in the backend and the cache.
-func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
 	if !autoCacheTypes(h) {
 		return b.Backend.Save(ctx, h, rd)
 	}
@@ -92,7 +91,7 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR
 	return nil
 }
 
-func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
+func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
 	finish := make(chan struct{})
 
 	b.inProgressMutex.Lock()
@@ -136,7 +135,7 @@ func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
 }
 
 // loadFromCache will try to load the file from the cache.
-func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
+func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
 	rd, inCache, err := b.Cache.load(h, length, offset)
 	if err != nil {
 		return inCache, err
@@ -151,7 +150,7 @@ func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, cons
 }
 
 // Load loads a file from the cache or the backend.
-func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
 	b.inProgressMutex.Lock()
 	waitForFinish, inProgress := b.inProgress[h]
 	b.inProgressMutex.Unlock()
@@ -198,7 +197,7 @@ func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset
 
 // Stat tests whether the backend has a file. If it does not exist but still
 // exists in the cache, it is removed from the cache.
-func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
+func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
 	debug.Log("cache Stat(%v)", h)
 
 	fi, err := b.Backend.Stat(ctx, h)
@@ -211,31 +210,24 @@ func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo,
 }
 
 // IsNotExist returns true if the error is caused by a non-existing file.
-func (b *Backend) IsNotExist(err error) bool {
+func (b *cacheBackend) IsNotExist(err error) bool {
 	return b.Backend.IsNotExist(err)
 }
 
-func (b *Backend) Unwrap() backend.Backend {
+func (b *cacheBackend) Unwrap() backend.Backend {
 	return b.Backend
 }
 
-func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
+func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
 	if !b.Cache.canBeCached(t) {
 		return b.Backend.List(ctx, t, fn)
 	}
 
-	// will contain the IDs of the files that are in the repository
-	ids := restic.NewIDSet()
+	ids := make(map[string]struct{})
 
 	// wrap the original function to also add the file to the ids set
 	wrapFn := func(f backend.FileInfo) error {
-		id, err := restic.ParseID(f.Name)
-		if err != nil {
-			// ignore files with invalid name
-			return nil
-		}
-
-		ids.Insert(id)
+		ids[f.Name] = struct{}{}
 
 		// execute the original function
 		return fn(f)
@@ -260,11 +252,11 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen
 }
 
 // Warmup delegates to wrapped backend.
-func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
+func (b *cacheBackend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
 	return b.Backend.Warmup(ctx, h)
 }
 
 // WarmupWait delegates to wrapped backend.
-func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error {
+func (b *cacheBackend) WarmupWait(ctx context.Context, h []backend.Handle) error {
 	return b.Backend.WarmupWait(ctx, h)
 }

internal/backend/cache/cache.go

@@ -12,7 +12,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/restic"
 )
 
 // Cache manages a local cache.
@@ -43,10 +42,10 @@ func readVersion(dir string) (v uint, err error) {
 
 const cacheVersion = 1
 
-var cacheLayoutPaths = map[restic.FileType]string{
-	restic.PackFile:     "data",
-	restic.SnapshotFile: "snapshots",
-	restic.IndexFile:    "index",
+var cacheLayoutPaths = map[backend.FileType]string{
+	backend.PackFile:     "data",
+	backend.SnapshotFile: "snapshots",
+	backend.IndexFile:    "index",
 }
 
 const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"

internal/backend/cache/file.go

@@ -12,7 +12,6 @@ import (
 	"github.com/restic/restic/internal/backend/util"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/restic"
 )
 
 func (c *Cache) filename(h backend.Handle) string {
@@ -171,7 +170,7 @@ func (c *Cache) remove(h backend.Handle) (bool, error) {
 
 // Clear removes all files of type t from the cache that are not contained in
 // the set valid.
-func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
+func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
 	debug.Log("Clearing cache for %v: %v valid files", t, len(valid))
 	if !c.canBeCached(t) {
 		return nil
@@ -183,12 +182,12 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
 	}
 
 	for id := range list {
-		if valid.Has(id) {
+		if _, ok := valid[id]; ok {
 			continue
 		}
 
 		// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
-		if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
+		if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id})); err != nil && !errors.Is(err, os.ErrNotExist) {
 			return err
 		}
 	}
@@ -201,12 +200,12 @@ func isFile(fi os.FileInfo) bool {
 }
 
 // list returns a list of all files of type T in the cache.
-func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
+func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
 	if !c.canBeCached(t) {
 		return nil, errors.New("cannot be cached")
 	}
 
-	list := restic.NewIDSet()
+	list := make(map[string]struct{})
 	dir := filepath.Join(c.path, cacheLayoutPaths[t])
 	err := filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
 		if err != nil {
@@ -221,12 +220,8 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
 			return nil
 		}
 
-		id, err := restic.ParseID(filepath.Base(name))
-		if err != nil {
-			return nil
-		}
-
-		list.Insert(id)
+		id := filepath.Base(name)
+		list[id] = struct{}{}
 		return nil
 	})
 

internal/backend/cache/file_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"maps"
 	"math/rand"
 	"os"
 	"runtime"
@@ -18,8 +19,8 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet {
-	ids := restic.NewIDSet()
+func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} {
+	ids := make(map[string]struct{})
 	for i := 0; i < random.Intn(15)+10; i++ {
 		buf := rtest.Random(random.Int(), 1<<19)
 		id := restic.Hash(buf)
@@ -33,13 +34,13 @@ func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType,
 		if err != nil {
 			t.Fatal(err)
 		}
-		ids.Insert(id)
+		ids[id.String()] = struct{}{}
 	}
 	return ids
 }
 
 // randomID returns a random ID from s.
-func randomID(s restic.IDSet) restic.ID {
+func randomID(s map[string]struct{}) string {
 	for id := range s {
 		return id
 	}
@@ -69,7 +70,7 @@ func load(t testing.TB, c *Cache, h backend.Handle) []byte {
 	return buf
 }
 
-func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
+func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{} {
 	list, err := c.list(tpe)
 	if err != nil {
 		t.Errorf("listing failed: %v", err)
@@ -78,7 +79,7 @@ func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
 	return list
 }
 
-func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
+func clearFiles(t testing.TB, c *Cache, tpe backend.FileType, valid map[string]struct{}) {
 	if err := c.Clear(tpe, valid); err != nil {
 		t.Error(err)
 	}
@@ -102,34 +103,34 @@ func TestFiles(t *testing.T) {
 			ids := generateRandomFiles(t, random, tpe, c)
 			id := randomID(ids)
 
-			h := backend.Handle{Type: tpe, Name: id.String()}
+			h := backend.Handle{Type: tpe, Name: id}
 			id2 := restic.Hash(load(t, c, h))
 
-			if !id.Equal(id2) {
-				t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
+			if id != id2.String() {
+				t.Errorf("wrong data returned, want %v, got %v", id, id2.String())
 			}
 
 			if !c.Has(h) {
-				t.Errorf("cache thinks index %v isn't present", id.Str())
+				t.Errorf("cache thinks index %v isn't present", id)
 			}
 
 			list := listFiles(t, c, tpe)
-			if !ids.Equals(list) {
+			if !maps.Equal(ids, list) {
 				t.Errorf("wrong list of index IDs returned, want:\n  %v\ngot:\n  %v", ids, list)
 			}
 
-			clearFiles(t, c, tpe, restic.NewIDSet(id))
+			clearFiles(t, c, tpe, map[string]struct{}{id: {}})
 			list2 := listFiles(t, c, tpe)
-			ids.Delete(id)
-			want := restic.NewIDSet(id)
-			if !list2.Equals(want) {
+			delete(ids, id)
+			want := map[string]struct{}{id: {}}
+			if !maps.Equal(list2, want) {
 				t.Errorf("ClearIndexes removed indexes, want:\n  %v\ngot:\n  %v", list2, want)
 			}
 
-			clearFiles(t, c, tpe, restic.NewIDSet())
-			want = restic.NewIDSet()
+			clearFiles(t, c, tpe, map[string]struct{}{})
+			want = map[string]struct{}{}
 			list3 := listFiles(t, c, tpe)
-			if !list3.Equals(want) {
+			if !maps.Equal(list3, want) {
 				t.Errorf("ClearIndexes returned a wrong list, want:\n  %v\ngot:\n  %v", want, list3)
 			}
 		})

internal/backend/cache/testing.go

@@ -3,16 +3,17 @@ package cache
 import (
 	"testing"
 
-	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
 )
 
+const testCacheID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
 // TestNewCache returns a cache in a temporary directory which is removed when
 // cleanup is called.
 func TestNewCache(t testing.TB) *Cache {
 	dir := test.TempDir(t)
 	t.Logf("created new cache at %v", dir)
-	cache, err := New(restic.NewRandomID().String(), dir)
+	cache, err := New(testCacheID, dir)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/backend/gs/gs.go

@@ -27,27 +27,25 @@ import (
 	"google.golang.org/api/option"
 )
 
-// Backend stores data in a GCS bucket.
+// gs stores data in a GCS bucket.
 //
 // The service account used to access the bucket must have these permissions:
 //   - storage.objects.create
 //   - storage.objects.delete
 //   - storage.objects.get
 //   - storage.objects.list
-type Backend struct {
-	gcsClient    *storage.Client
-	projectID    string
-	connections  uint
-	bucketName   string
-	region       string
-	bucket       *storage.BucketHandle
-	prefix       string
-	listMaxItems int
+type gs struct {
+	gcsClient   *storage.Client
+	projectID   string
+	connections uint
+	bucketName  string
+	region      string
+	bucket      *storage.BucketHandle
 	layout.Layout
 }
 
 // Ensure that *Backend implements backend.Backend.
-var _ backend.Backend = &Backend{}
+var _ backend.Backend = &gs{}
 
 func NewFactory() location.Factory {
 	return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@@ -86,7 +84,7 @@ func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
 	return gcsClient, nil
 }
 
-func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
+func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
 	_, err := bucket.Attrs(ctx)
 	if err == storage.ErrBucketNotExist {
 		return false, nil
@@ -94,9 +92,7 @@ func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandl
 	return err == nil, err
 }
 
-const defaultListMaxItems = 1000
-
-func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
+func open(cfg Config, rt http.RoundTripper) (*gs, error) {
 	debug.Log("open, config %#v", cfg)
 
 	gcsClient, err := getStorageClient(rt)
@@ -104,16 +100,14 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 		return nil, errors.Wrap(err, "getStorageClient")
 	}
 
-	be := &Backend{
-		gcsClient:    gcsClient,
-		projectID:    cfg.ProjectID,
-		connections:  cfg.Connections,
-		bucketName:   cfg.Bucket,
-		region:       cfg.Region,
-		bucket:       gcsClient.Bucket(cfg.Bucket),
-		prefix:       cfg.Prefix,
-		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
-		listMaxItems: defaultListMaxItems,
+	be := &gs{
+		gcsClient:   gcsClient,
+		projectID:   cfg.ProjectID,
+		connections: cfg.Connections,
+		bucketName:  cfg.Bucket,
+		region:      cfg.Region,
+		bucket:      gcsClient.Bucket(cfg.Bucket),
+		Layout:      layout.NewDefaultLayout(cfg.Prefix, path.Join),
 	}
 
 	return be, nil
@@ -161,17 +155,12 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
 	return be, nil
 }
 
-// SetListMaxItems sets the number of list items to load per request.
-func (be *Backend) SetListMaxItems(i int) {
-	be.listMaxItems = i
-}
-
 // IsNotExist returns true if the error is caused by a not existing file.
-func (be *Backend) IsNotExist(err error) bool {
+func (be *gs) IsNotExist(err error) bool {
 	return errors.Is(err, storage.ErrObjectNotExist)
 }
 
-func (be *Backend) IsPermanentError(err error) bool {
+func (be *gs) IsPermanentError(err error) bool {
 	if be.IsNotExist(err) {
 		return true
 	}
@@ -186,7 +175,7 @@ func (be *Backend) IsPermanentError(err error) bool {
 	return false
 }
 
-func (be *Backend) Properties() backend.Properties {
+func (be *gs) Properties() backend.Properties {
 	return backend.Properties{
 		Connections:      be.connections,
 		HasAtomicReplace: true,
@@ -194,17 +183,12 @@ func (be *Backend) Properties() backend.Properties {
 }
 
 // Hasher may return a hash function for calculating a content hash for the backend
-func (be *Backend) Hasher() hash.Hash {
+func (be *gs) Hasher() hash.Hash {
 	return md5.New()
 }
 
-// Path returns the path in the bucket that is used for this backend.
-func (be *Backend) Path() string {
-	return be.prefix
-}
-
 // Save stores data in the backend at the handle.
-func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
 	objName := be.Filename(h)
 
 	// Set chunk size to zero to disable resumable uploads.
@@ -254,14 +238,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
 
 // Load runs fn with a reader that yields the contents of the file at h at the
 // given offset.
-func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+func (be *gs) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
 }
 
-func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
+func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
 	if length == 0 {
 		// negative length indicates read till end to GCS lib
 		length = -1
@@ -283,7 +267,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int,
 }
 
 // Stat returns information about a blob.
-func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
+func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
 	objName := be.Filename(h)
 
 	attr, err := be.bucket.Object(objName).Attrs(ctx)
@@ -296,7 +280,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI
 }
 
 // Remove removes the blob with the given name and type.
-func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
+func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
 	objName := be.Filename(h)
 
 	err := be.bucket.Object(objName).Delete(ctx)
@@ -310,7 +294,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
 
 // List runs fn for each file in the backend which has the type t. When an
 // error occurs (or fn returns an error), List stops and returns it.
-func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
+func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
 	prefix, _ := be.Basedir(t)
 
 	// make sure prefix ends with a slash
@@ -355,15 +339,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
 }
 
 // Delete removes all restic keys in the bucket. It will not remove the bucket itself.
-func (be *Backend) Delete(ctx context.Context) error {
+func (be *gs) Delete(ctx context.Context) error {
 	return util.DefaultDelete(ctx, be)
 }
 
 // Close does nothing.
-func (be *Backend) Close() error { return nil }
+func (be *gs) Close() error { return nil }
 
 // Warmup not implemented
-func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
+func (be *gs) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
 	return []backend.Handle{}, nil
 }
-func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
+func (be *gs) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }

internal/backend/rclone/backend.go

@@ -28,8 +28,8 @@ import (
 	"golang.org/x/net/http2"
 )
 
-// Backend is used to access data stored somewhere via rclone.
-type Backend struct {
+// rclone is used to access data stored somewhere via rclone.
+type rclone struct {
 	*rest.Backend
 	tr         *http2.Transport
 	cmd        *exec.Cmd
@@ -141,7 +141,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn {
 }
 
 // New initializes a Backend and starts the process.
-func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
+func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*rclone, error) {
 	var (
 		args []string
 		err  error
@@ -196,7 +196,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
 	}
 
 	cmd := stdioConn.cmd
-	be := &Backend{
+	be := &rclone{
 		tr:     tr,
 		cmd:    cmd,
 		waitCh: waitCh,
@@ -270,7 +270,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
 }
 
 // Open starts an rclone process with the given config.
-func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
+func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
 	be, err := newBackend(ctx, cfg, lim, errorLog)
 	if err != nil {
 		return nil, err
@@ -297,7 +297,7 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(st
 }
 
 // Create initializes a new restic repo with rclone.
-func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
+func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
 	be, err := newBackend(ctx, cfg, lim, errorLog)
 	if err != nil {
 		return nil, err
@@ -328,7 +328,7 @@ func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(
 const waitForExit = 5 * time.Second
 
 // Close terminates the backend.
-func (be *Backend) Close() error {
+func (be *rclone) Close() error {
 	debug.Log("exiting rclone")
 	be.tr.CloseIdleConnections()
 
@@ -348,7 +348,7 @@ func (be *Backend) Close() error {
 	return be.waitResult
 }
 
-func (be *Backend) Properties() backend.Properties {
+func (be *rclone) Properties() backend.Properties {
 	properties := be.Backend.Properties()
 	properties.HasFlakyErrors = true
 	return properties

internal/backend/rclone/internal_test.go

@@ -27,7 +27,7 @@ func TestRcloneExit(t *testing.T) {
 		_ = be.Close()
 	}()
 
-	err = be.cmd.Process.Kill()
+	err = be.(*rclone).cmd.Process.Kill()
 	rtest.OK(t, err)
 	t.Log("killed rclone")
 

internal/backend/s3/s3.go

@@ -25,15 +25,15 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
-// Backend stores data on an S3 endpoint.
-type Backend struct {
+// s3 stores data on an S3 endpoint.
+type s3 struct {
 	client *minio.Client
 	cfg    Config
 	layout.Layout
 }
 
 // make sure that *Backend implements backend.Backend
-var _ backend.Backend = &Backend{}
+var _ backend.Backend = &s3{}
 
 var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"}
 
@@ -50,7 +50,7 @@ func NewFactory() location.Factory {
 	return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
 }
 
-func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
+func open(cfg Config, rt http.RoundTripper) (*s3, error) {
 	debug.Log("open, config %#v", cfg)
 
 	if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) {
@@ -89,7 +89,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 		return nil, errors.Wrap(err, "minio.New")
 	}
 
-	be := &Backend{
+	be := &s3{
 		client: client,
 		cfg:    cfg,
 		Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
@@ -240,12 +240,12 @@ func isAccessDenied(err error) bool {
 }
 
 // IsNotExist returns true if the error is caused by a not existing file.
-func (be *Backend) IsNotExist(err error) bool {
+func (be *s3) IsNotExist(err error) bool {
 	var e minio.ErrorResponse
 	return errors.As(err, &e) && e.Code == "NoSuchKey"
 }
 
-func (be *Backend) IsPermanentError(err error) bool {
+func (be *s3) IsPermanentError(err error) bool {
 	if be.IsNotExist(err) {
 		return true
 	}
@@ -260,7 +260,7 @@ func (be *Backend) IsPermanentError(err error) bool {
 	return false
 }
 
-func (be *Backend) Properties() backend.Properties {
+func (be *s3) Properties() backend.Properties {
 	return backend.Properties{
 		Connections:      be.cfg.Connections,
 		HasAtomicReplace: true,
@@ -268,26 +268,21 @@ func (be *Backend) Properties() backend.Properties {
 }
 
 // Hasher may return a hash function for calculating a content hash for the backend
-func (be *Backend) Hasher() hash.Hash {
+func (be *s3) Hasher() hash.Hash {
 	return nil
 }
 
-// Path returns the path in the bucket that is used for this backend.
-func (be *Backend) Path() string {
-	return be.cfg.Prefix
-}
-
 // useStorageClass returns whether file should be saved in the provided Storage Class
 // For archive storage classes, only data files are stored using that class; metadata
 // must remain instantly accessible.
-func (be *Backend) useStorageClass(h backend.Handle) bool {
+func (be *s3) useStorageClass(h backend.Handle) bool {
 	isDataFile := h.Type == backend.PackFile && !h.IsMetadata
 	isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass)
 	return !isArchiveClass || isDataFile
 }
 
 // Save stores data in the backend at the handle.
-func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
 	objName := be.Filename(h)
 
 	opts := minio.PutObjectOptions{
@@ -313,14 +308,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
 
 // Load runs fn with a reader that yields the contents of the file at h at the
 // given offset.
-func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+func (be *s3) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
 }
 
-func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
+func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
 	objName := be.Filename(h)
 	opts := minio.GetObjectOptions{}
 
@@ -352,7 +347,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int,
 }
 
 // Stat returns information about a blob.
-func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
+func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
 	objName := be.Filename(h)
 	var obj *minio.Object
 
@@ -380,7 +375,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI
 }
 
 // Remove removes the blob with the given name and type.
-func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
+func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
 	objName := be.Filename(h)
 
 	err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@@ -394,7 +389,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
 
 // List runs fn for each file in the backend which has the type t. When an
 // error occurs (or fn returns an error), List stops and returns it.
-func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
+func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
 	prefix, recursive := be.Basedir(t)
 
 	// make sure prefix ends with a slash
@@ -449,15 +444,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
 }
 
 // Delete removes all restic keys in the bucket. It will not remove the bucket itself.
-func (be *Backend) Delete(ctx context.Context) error {
+func (be *s3) Delete(ctx context.Context) error {
 	return util.DefaultDelete(ctx, be)
 }
 
 // Close does nothing
-func (be *Backend) Close() error { return nil }
+func (be *s3) Close() error { return nil }
 
 // Warmup transitions handles from cold to hot storage if needed.
-func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
+func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
 	handlesWarmingUp := []backend.Handle{}
 
 	if be.cfg.EnableRestore {
@@ -478,7 +473,7 @@ func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]back
 }
 
 // requestRestore sends a glacier restore request on a given file.
-func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) {
+func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error) {
 	objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{})
 	if err != nil {
 		return false, err
@@ -514,7 +509,7 @@ func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, e
 }
 
 // getWarmupStatus returns the warmup status of the provided object.
-func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
+func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
 	// We can't use objectInfo.StorageClass to get the storage class of the
 	// object because this field is only set during ListObjects operations.
 	// The response header is the documented way to get the storage class
@@ -545,7 +540,7 @@ func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
 }
 
 // WarmupWait waits until all handles are in hot storage.
-func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error {
+func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
 	timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout)
 	defer timeoutCtxCancel()
 
@@ -564,7 +559,7 @@ func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) err
 }
 
 // waitForRestore waits for a given file to be restored.
-func (be *Backend) waitForRestore(ctx context.Context, filename string) error {
+func (be *s3) waitForRestore(ctx context.Context, filename string) error {
 	for {
 		var objectInfo minio.ObjectInfo
 

internal/backend/test/tests.go

@@ -251,10 +251,6 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
 	test.OK(t, b.Remove(context.TODO(), handle))
 }
 
-type setter interface {
-	SetListMaxItems(int)
-}
-
 // TestList makes sure that the backend implements List() pagination correctly.
 func (s *Suite[C]) TestList(t *testing.T) {
 	random := seedRand(t)
@@ -302,54 +298,39 @@ func (s *Suite[C]) TestList(t *testing.T) {
 
 	t.Logf("wrote %v files", len(list1))
 
-	var tests = []struct {
-		maxItems int
-	}{
-		{11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123},
+	list2 := make(map[restic.ID]int64)
+
+	err = b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
+		id, err := restic.ParseID(fi.Name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		list2[id] = fi.Size
+		return nil
+	})
+
+	if err != nil {
+		t.Fatalf("List returned error %v", err)
 	}
 
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
-			list2 := make(map[restic.ID]int64)
+	t.Logf("loaded %v IDs from backend", len(list2))
 
-			if s, ok := b.(setter); ok {
-				t.Logf("setting max list items to %d", test.maxItems)
-				s.SetListMaxItems(test.maxItems)
-			}
+	for id, size := range list1 {
+		size2, ok := list2[id]
+		if !ok {
+			t.Errorf("id %v not returned by List()", id.Str())
+		}
 
-			err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
-				id, err := restic.ParseID(fi.Name)
-				if err != nil {
-					t.Fatal(err)
-				}
-				list2[id] = fi.Size
-				return nil
-			})
+		if size != size2 {
+			t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
+		}
+	}
 
-			if err != nil {
-				t.Fatalf("List returned error %v", err)
-			}
-
-			t.Logf("loaded %v IDs from backend", len(list2))
-
-			for id, size := range list1 {
-				size2, ok := list2[id]
-				if !ok {
-					t.Errorf("id %v not returned by List()", id.Str())
-				}
-
-				if size != size2 {
-					t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
-				}
-			}
-
-			for id := range list2 {
-				_, ok := list1[id]
-				if !ok {
-					t.Errorf("extra id %v returned by List()", id.Str())
-				}
-			}
-		})
+	for id := range list2 {
+		_, ok := list1[id]
+		if !ok {
+			t.Errorf("extra id %v returned by List()", id.Str())
+		}
 	}
 
 	t.Logf("remove %d files", numTestFiles)

internal/checker/checker_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"io"
 	"math/rand"
-	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -19,7 +18,6 @@ import (
 	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/repository/hashing"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
 )
@@ -111,7 +109,7 @@ func TestMissingPack(t *testing.T) {
 	test.Assert(t, len(errs) == 1,
 		"expected exactly one error, got %v", len(errs))
 
-	if err, ok := errs[0].(*repository.PackError); ok {
+	if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
 		test.Equals(t, packID, err.ID)
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -139,7 +137,7 @@ func TestUnreferencedPack(t *testing.T) {
 	test.Assert(t, len(errs) == 1,
 		"expected exactly one error, got %v", len(errs))
 
-	if err, ok := errs[0].(*repository.PackError); ok {
+	if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
 		test.Equals(t, packID, err.ID.String())
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -192,56 +190,19 @@ func TestModifiedIndex(t *testing.T) {
 		Type: restic.IndexFile,
 		Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
 	}
-
-	tmpfile, err := os.CreateTemp("", "restic-test-mod-index-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer func() {
-		err := tmpfile.Close()
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		err = os.Remove(tmpfile.Name())
-		if err != nil {
-			t.Fatal(err)
-		}
-	}()
-	wr := io.Writer(tmpfile)
-	var hw *hashing.Writer
-	if be.Hasher() != nil {
-		hw = hashing.NewWriter(wr, be.Hasher())
-		wr = hw
-	}
-
-	// read the file from the backend
-	err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
-		_, err := io.Copy(wr, rd)
+	var data []byte
+	test.OK(t, be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
+		var err error
+		data, err = io.ReadAll(rd)
 		return err
-	})
-	test.OK(t, err)
-
+	}))
 	// save the index again with a modified name so that the hash doesn't match
 	// the content any more
 	h2 := backend.Handle{
 		Type: restic.IndexFile,
 		Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
 	}
-
-	var hash []byte
-	if hw != nil {
-		hash = hw.Sum(nil)
-	}
-	rd, err := backend.NewFileReader(tmpfile, hash)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = be.Save(context.TODO(), h2, rd)
-	if err != nil {
-		t.Fatal(err)
-	}
+	test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher())))
 
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)

internal/global/global.go

@@ -35,7 +35,7 @@ import (
 // to a missing backend storage location or config file
 var ErrNoRepository = errors.New("repository does not exist")
 
-const Version = "0.19.0"
+const Version = "0.19.0-dev (compiled manually)"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"

internal/repository/check.go

@@ -1,215 +0,0 @@
-package repository
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"fmt"
-	"io"
-
-	"github.com/klauspost/compress/zstd"
-	"github.com/restic/restic/internal/backend"
-	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository/hashing"
-	"github.com/restic/restic/internal/repository/pack"
-	"github.com/restic/restic/internal/restic"
-)
-
-// ErrPackData is returned if errors are discovered while verifying a packfile
-type ErrPackData struct {
-	PackID restic.ID
-	errs   []error
-}
-
-func (e *ErrPackData) Error() string {
-	return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
-}
-
-type partialReadError struct {
-	err error
-}
-
-func (e *partialReadError) Error() string {
-	return e.err.Error()
-}
-
-// CheckPack reads a pack and checks the integrity of all blobs.
-func CheckPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
-	err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
-	if err != nil {
-		if r.cache != nil {
-			// ignore error as there's not much we can do here
-			_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
-		}
-
-		// retry pack verification to detect transient errors
-		err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
-		if err2 != nil {
-			err = err2
-		} else {
-			err = fmt.Errorf("check successful on second attempt, original error %w", err)
-		}
-	}
-	return err
-}
-
-func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
-
-	debug.Log("checking pack %v", id.String())
-
-	if len(blobs) == 0 {
-		return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
-	}
-
-	// sanity check blobs in index
-	blobs.Sort()
-	idxHdrSize := pack.CalculateHeaderSize(blobs)
-	lastBlobEnd := 0
-	nonContinuousPack := false
-	for _, blob := range blobs {
-		if lastBlobEnd != int(blob.Offset) {
-			nonContinuousPack = true
-		}
-		lastBlobEnd = int(blob.Offset + blob.Length)
-	}
-	// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
-
-	var errs []error
-	if nonContinuousPack {
-		debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
-		errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
-	}
-
-	// calculate hash on-the-fly while reading the pack and capture pack header
-	var hash restic.ID
-	var hdrBuf []byte
-	// must use a separate slice from `errs` here as we're only interested in the last retry
-	var blobErrors []error
-	h := backend.Handle{Type: backend.PackFile, Name: id.String()}
-	err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
-		hrd := hashing.NewReader(rd, sha256.New())
-		bufRd.Reset(hrd)
-		// reset blob errors for each retry
-		blobErrors = nil
-
-		it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
-		for {
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
-
-			val, err := it.Next()
-			if err == errPackEOF {
-				break
-			} else if err != nil {
-				return &partialReadError{err}
-			}
-			debug.Log("  check blob %v: %v", val.Handle.ID, val.Handle)
-			if val.Err != nil {
-				debug.Log("  error verifying blob %v: %v", val.Handle.ID, val.Err)
-				blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
-			}
-		}
-
-		// skip enough bytes until we reach the possible header start
-		curPos := lastBlobEnd
-		minHdrStart := int(size) - pack.MaxHeaderSize
-		if minHdrStart > curPos {
-			_, err := bufRd.Discard(minHdrStart - curPos)
-			if err != nil {
-				return &partialReadError{err}
-			}
-			curPos += minHdrStart - curPos
-		}
-
-		// read remainder, which should be the pack header
-		var err error
-		hdrBuf = make([]byte, int(size-int64(curPos)))
-		_, err = io.ReadFull(bufRd, hdrBuf)
-		if err != nil {
-			return &partialReadError{err}
-		}
-
-		hash = restic.IDFromHash(hrd.Sum(nil))
-		return nil
-	})
-	errs = append(errs, blobErrors...)
-	if err != nil {
-		var e *partialReadError
-		isPartialReadError := errors.As(err, &e)
-		// failed to load the pack file, return as further checks cannot succeed anyways
-		debug.Log("  error streaming pack (partial %v): %v", isPartialReadError, err)
-		if isPartialReadError {
-			return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
-		}
-
-		// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
-		// completely failed to download such that there's no point in repairing anything.
-		return fmt.Errorf("download error: %w", err)
-	}
-	if !hash.Equal(id) {
-		debug.Log("pack ID does not match, want %v, got %v", id, hash)
-		return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
-	}
-
-	blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
-	if err != nil {
-		return &ErrPackData{PackID: id, errs: append(errs, err)}
-	}
-
-	if uint32(idxHdrSize) != hdrSize {
-		debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
-		errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
-	}
-
-	for _, blob := range blobs {
-		// Check if blob is contained in index and position is correct
-		idxHas := false
-		for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
-			if pb.PackID == id && pb.Blob == blob {
-				idxHas = true
-				break
-			}
-		}
-		if !idxHas {
-			errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
-			continue
-		}
-	}
-
-	if len(errs) > 0 {
-		return &ErrPackData{PackID: id, errs: errs}
-	}
-
-	return nil
-}
-
-type bufReader struct {
-	rd  *bufio.Reader
-	buf []byte
-}
-
-func newBufReader(rd *bufio.Reader) *bufReader {
-	return &bufReader{
-		rd: rd,
-	}
-}
-
-func (b *bufReader) Discard(n int) (discarded int, err error) {
-	return b.rd.Discard(n)
-}
-
-func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
-	if cap(b.buf) < n {
-		b.buf = make([]byte, n)
-	}
-	b.buf = b.buf[:n]
-
-	_, err = io.ReadFull(b.rd, b.buf)
-	if err != nil {
-		return nil, err
-	}
-	return b.buf, nil
-}

internal/repository/checker.go

@@ -2,12 +2,17 @@ package repository
 
 import (
 	"bufio"
+	"bytes"
 	"context"
+	"crypto/sha256"
 	"fmt"
+	"io"
 
 	"github.com/klauspost/compress/zstd"
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository/hashing"
 	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
@@ -46,18 +51,29 @@ func (e *ErrMixedPack) Error() string {
 	return fmt.Sprintf("pack %v contains a mix of tree and data blobs", e.PackID.Str())
 }
 
-// PackError describes an error with a specific pack.
-type PackError struct {
+// ErrPackMetadata describes an error with a specific pack. It is used for missing, truncated or orphaned packs.
+// Errors of the actual pack data are returned as ErrPackData.
+type ErrPackMetadata struct {
 	ID        restic.ID
 	Orphaned  bool
 	Truncated bool
 	Err       error
 }
 
-func (e *PackError) Error() string {
+func (e *ErrPackMetadata) Error() string {
 	return "pack " + e.ID.String() + ": " + e.Err.Error()
 }
 
+// ErrPackData is returned if errors are discovered while verifying a packfile
+type ErrPackData struct {
+	PackID restic.ID
+	errs   []error
+}
+
+func (e *ErrPackData) Error() string {
+	return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
+}
+
 // Checker handles index-related operations for repository checking.
 type Checker struct {
 	repo *Repository
@@ -199,7 +215,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
 			select {
 			case <-ctx.Done():
 				return
-			case errChan <- &PackError{ID: id, Err: errors.New("does not exist")}:
+			case errChan <- &ErrPackMetadata{ID: id, Err: errors.New("does not exist")}:
 			}
 			continue
 		}
@@ -209,7 +225,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
 			select {
 			case <-ctx.Done():
 				return
-			case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
+			case errChan <- &ErrPackMetadata{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
 			}
 		}
 	}
@@ -219,7 +235,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
 		select {
 		case <-ctx.Done():
 			return
-		case errChan <- &PackError{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
+		case errChan <- &ErrPackMetadata{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
 		}
 	}
 }
@@ -269,7 +285,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
 					}
 				}
 
-				err := CheckPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
+				err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
 				p.Add(1)
 				if err == nil {
 					continue
@@ -309,3 +325,186 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
 		}
 	}
 }
+
+// checkPack reads a pack and checks the integrity of all blobs.
+func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
+	err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
+	if err != nil {
+		if r.cache != nil {
+			// ignore error as there's not much we can do here
+			_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
+		}
+
+		// retry pack verification to detect transient errors
+		err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
+		if err2 != nil {
+			err = err2
+		} else {
+			err = fmt.Errorf("check successful on second attempt, original error %w", err)
+		}
+	}
+	return err
+}
+
+func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
+
+	type partialReadError struct {
+		error
+	}
+
+	debug.Log("checking pack %v", id.String())
+
+	if len(blobs) == 0 {
+		return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
+	}
+
+	// sanity check blobs in index
+	blobs.Sort()
+	idxHdrSize := pack.CalculateHeaderSize(blobs)
+	lastBlobEnd := 0
+	nonContinuousPack := false
+	for _, blob := range blobs {
+		if lastBlobEnd != int(blob.Offset) {
+			nonContinuousPack = true
+		}
+		lastBlobEnd = int(blob.Offset + blob.Length)
+	}
+	// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
+
+	var errs []error
+	if nonContinuousPack {
+		debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
+		errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
+	}
+
+	// calculate hash on-the-fly while reading the pack and capture pack header
+	var hash restic.ID
+	var hdrBuf []byte
+	// must use a separate slice from `errs` here as we're only interested in the last retry
+	var blobErrors []error
+	h := backend.Handle{Type: backend.PackFile, Name: id.String()}
+	err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
+		hrd := hashing.NewReader(rd, sha256.New())
+		bufRd.Reset(hrd)
+		// reset blob errors for each retry
+		blobErrors = nil
+
+		it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
+		for {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+
+			val, err := it.Next()
+			if err == errPackEOF {
+				break
+			} else if err != nil {
+				return &partialReadError{err}
+			}
+			debug.Log("  check blob %v: %v", val.Handle.ID, val.Handle)
+			if val.Err != nil {
+				debug.Log("  error verifying blob %v: %v", val.Handle.ID, val.Err)
+				blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
+			}
+		}
+
+		// skip enough bytes until we reach the possible header start
+		curPos := lastBlobEnd
+		minHdrStart := int(size) - pack.MaxHeaderSize
+		if minHdrStart > curPos {
+			_, err := bufRd.Discard(minHdrStart - curPos)
+			if err != nil {
+				return &partialReadError{err}
+			}
+			curPos += minHdrStart - curPos
+		}
+
+		// read remainder, which should be the pack header
+		var err error
+		hdrBuf = make([]byte, int(size-int64(curPos)))
+		_, err = io.ReadFull(bufRd, hdrBuf)
+		if err != nil {
+			return &partialReadError{err}
+		}
+
+		hash = restic.IDFromHash(hrd.Sum(nil))
+		return nil
+	})
+	errs = append(errs, blobErrors...)
+	if err != nil {
+		var e *partialReadError
+		isPartialReadError := errors.As(err, &e)
+		// failed to load the pack file, return as further checks cannot succeed anyways
+		debug.Log("  error streaming pack (partial %v): %v", isPartialReadError, err)
+		if isPartialReadError {
+			return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
+		}
+
+		// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
+		// completely failed to download such that there's no point in repairing anything.
+		return fmt.Errorf("download error: %w", err)
+	}
+	if !hash.Equal(id) {
+		debug.Log("pack ID does not match, want %v, got %v", id, hash)
+		return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
+	}
+
+	blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
+	if err != nil {
+		return &ErrPackData{PackID: id, errs: append(errs, err)}
+	}
+
+	if uint32(idxHdrSize) != hdrSize {
+		debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
+		errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
+	}
+
+	for _, blob := range blobs {
+		// Check if blob is contained in index and position is correct
+		idxHas := false
+		for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
+			if pb.PackID == id && pb.Blob == blob {
+				idxHas = true
+				break
+			}
+		}
+		if !idxHas {
+			errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
+			continue
+		}
+	}
+
+	if len(errs) > 0 {
+		return &ErrPackData{PackID: id, errs: errs}
+	}
+
+	return nil
+}
+
+type bufReader struct {
+	rd  *bufio.Reader
+	buf []byte
+}
+
+func newBufReader(rd *bufio.Reader) *bufReader {
+	return &bufReader{
+		rd: rd,
+	}
+}
+
+func (b *bufReader) Discard(n int) (discarded int, err error) {
+	return b.rd.Discard(n)
+}
+
+func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
+	if cap(b.buf) < n {
+		b.buf = make([]byte, n)
+	}
+	b.buf = b.buf[:n]
+
+	_, err = io.ReadFull(b.rd, b.buf)
+	if err != nil {
+		return nil, err
+	}
+	return b.buf, nil
+}

internal/repository/debug.go

@@ -0,0 +1,390 @@
+//go:build debug
+
+package repository
+
+import (
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/klauspost/compress/zstd"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/restic/restic/internal/crypto"
+	"github.com/restic/restic/internal/repository/index"
+	"github.com/restic/restic/internal/repository/pack"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/progress"
+)
+
+type packDumpEntry struct {
+	Name  string         `json:"name"`
+	Blobs []packDumpBlob `json:"blobs"`
+}
+
+type packDumpBlob struct {
+	Type   restic.BlobType `json:"type"`
+	Length uint            `json:"length"`
+	ID     restic.ID       `json:"id"`
+	Offset uint            `json:"offset"`
+}
+
+func writePackDumpJSON(wr io.Writer, item any) error {
+	buf, err := json.MarshalIndent(item, "", "  ")
+	if err != nil {
+		return err
+	}
+	_, err = wr.Write(append(buf, '\n'))
+	return err
+}
+
+// DumpPacks lists each pack file and writes its header blob layout as JSON to wr.
+func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
+	var m sync.Mutex
+	return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+		blobs, err := repo.ListPack(ctx, id, size)
+		if err != nil {
+			printer.E("error for pack %v: %v", id.Str(), err)
+			return nil
+		}
+
+		p := packDumpEntry{
+			Name:  id.String(),
+			Blobs: make([]packDumpBlob, len(blobs)),
+		}
+		for i, blob := range blobs {
+			p.Blobs[i] = packDumpBlob{
+				Type:   blob.Type,
+				Length: blob.Length,
+				ID:     blob.ID,
+				Offset: blob.Offset,
+			}
+		}
+
+		m.Lock()
+		defer m.Unlock()
+		return writePackDumpJSON(wr, p)
+	})
+}
+
+// DumpIndexes loads each on-disk index file and writes its debug dump to wr.
+func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
+	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
+		printer.S("index_id: %v", id)
+		if err != nil {
+			return err
+		}
+
+		return idx.Dump(wr)
+	})
+}
+
+// ExaminePackOptions configures debug examination of a pack file.
+type ExaminePackOptions struct {
+	TryRepair     bool
+	RepairByte    bool
+	ExtractPack   bool
+	ReuploadBlobs bool
+}
+
+// ExaminePack loads and inspects a pack file and its index entries.
+func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error {
+	printer.S("examine %v", id)
+
+	buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
+	// also process damaged pack files
+	if buf == nil {
+		return err
+	}
+	printer.S("  file size is %v", len(buf))
+	gotID := restic.Hash(buf)
+	if !id.Equal(gotID) {
+		printer.S("  wanted hash %v, got %v", id, gotID)
+	} else {
+		printer.S("  hash for file content matches")
+	}
+
+	printer.S("  ========================================")
+	printer.S("  looking for info in the indexes")
+
+	blobsLoaded := false
+	// examine all data the indexes have for the pack file
+	for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
+		blobs := b.Blobs
+		if len(blobs) == 0 {
+			continue
+		}
+
+		checkPackSize(blobs, len(buf), printer)
+
+		err = loadBlobs(ctx, opts, repo, id, blobs, printer)
+		if err != nil {
+			printer.E("error: %v", err)
+		} else {
+			blobsLoaded = true
+		}
+	}
+
+	printer.S("  ========================================")
+	printer.S("  inspect the pack itself")
+
+	blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
+	if err != nil {
+		return fmt.Errorf("pack %v: %v", id.Str(), err)
+	}
+	checkPackSize(blobs, len(buf), printer)
+
+	if !blobsLoaded {
+		return loadBlobs(ctx, opts, repo, id, blobs, printer)
+	}
+	return nil
+}
+
+func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
+	// track current size and offset
+	var size, offset uint64
+
+	blobs.Sort()
+
+	for _, pb := range blobs {
+		printer.S("      %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
+		if offset != uint64(pb.Offset) {
+			printer.S("      hole in file, want offset %v, got %v", offset, pb.Offset)
+		}
+		offset = uint64(pb.Offset + pb.Length)
+		size += uint64(pb.Length)
+	}
+	size += uint64(pack.CalculateHeaderSize(blobs))
+
+	if uint64(fileSize) != size {
+		printer.S("      file sizes do not match: computed %v, file size is %v", size, fileSize)
+	} else {
+		printer.S("      file sizes match")
+	}
+}
+
+func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
+	if bytewise {
+		printer.S("        trying to repair blob by finding a broken byte")
+	} else {
+		printer.S("        trying to repair blob with single bit flip")
+	}
+
+	ch := make(chan int)
+	var wg errgroup.Group
+	done := make(chan struct{})
+	var fixed []byte
+	var found bool
+
+	workers := runtime.GOMAXPROCS(0)
+	printer.S("         spinning up %d worker functions", runtime.GOMAXPROCS(0))
+	for i := 0; i < workers; i++ {
+		wg.Go(func() error {
+			// make a local copy of the buffer
+			buf := make([]byte, len(input))
+			copy(buf, input)
+
+			testFlip := func(idx int, pattern byte) bool {
+				// flip bits
+				buf[idx] ^= pattern
+
+				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
+				if err == nil {
+					printer.S("")
+					printer.S("        blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
+					printer.S("        hash is %v", restic.Hash(plaintext))
+					close(done)
+					found = true
+					fixed = plaintext
+					return true
+				}
+
+				// flip bits back
+				buf[idx] ^= pattern
+				return false
+			}
+
+			for i := range ch {
+				if bytewise {
+					for j := 0; j < 255; j++ {
+						if testFlip(i, byte(j)) {
+							return nil
+						}
+					}
+				} else {
+					for j := 0; j < 7; j++ {
+						// flip each bit once
+						if testFlip(i, (1 << uint(j))) {
+							return nil
+						}
+					}
+				}
+			}
+			return nil
+		})
+	}
+
+	wg.Go(func() error {
+		defer close(ch)
+
+		start := time.Now()
+		info := time.Now()
+		for i := range input {
+			select {
+			case ch <- i:
+			case <-done:
+				printer.S("     done after %v", time.Since(start))
+				return nil
+			}
+
+			if time.Since(info) > time.Second {
+				secs := time.Since(start).Seconds()
+				gps := float64(i) / secs
+				remaining := len(input) - i
+				eta := time.Duration(float64(remaining)/gps) * time.Second
+
+				printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
+					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
+				info = time.Now()
+			}
+		}
+		return nil
+	})
+	err := wg.Wait()
+	if err != nil {
+		panic("all go routines can only return nil")
+	}
+
+	if !found {
+		printer.S("\n        blob could not be repaired")
+	}
+	return fixed
+}
+
+func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
+	// strip signature at the end
+	l := len(buf)
+	nonce, ct := buf[:16], buf[16:l-16]
+	out := make([]byte, len(ct))
+
+	c, err := aes.NewCipher(k.EncryptionKey[:])
+	if err != nil {
+		panic(fmt.Sprintf("unable to create cipher: %v", err))
+	}
+	e := cipher.NewCTR(c, nonce)
+	e.XORKeyStream(out, ct)
+
+	return out
+}
+
+func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
+	dec, err := zstd.NewReader(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	packData, err := repo.LoadRaw(ctx, restic.PackFile, packID)
+	// allow processing broken pack files
+	if packData == nil {
+		return err
+	}
+
+	err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
+		for _, blob := range list {
+			printer.S("      loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
+			if int(blob.Offset+blob.Length) > len(packData) {
+				printer.E("skipping truncated blob")
+				continue
+			}
+			buf := packData[blob.Offset : blob.Offset+blob.Length]
+			key := repo.Key()
+
+			nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+			plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+			outputPrefix := ""
+			filePrefix := ""
+			if err != nil {
+				printer.E("error decrypting blob: %v", err)
+				if opts.TryRepair || opts.RepairByte {
+					plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
+				}
+				if plaintext != nil {
+					outputPrefix = "repaired "
+					filePrefix = "repaired-"
+				} else {
+					plaintext = decryptUnsigned(key, buf)
+					err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
+					if err != nil {
+						return err
+					}
+					continue
+				}
+			}
+
+			if blob.IsCompressed() {
+				decompressed, err := dec.DecodeAll(plaintext, nil)
+				if err != nil {
+					printer.S("         failed to decompress blob %v", blob.ID)
+				}
+				if decompressed != nil {
+					plaintext = decompressed
+				}
+			}
+
+			id := restic.Hash(plaintext)
+			var prefix string
+			if !id.Equal(blob.ID) {
+				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
+				prefix = "wrong-hash-"
+			} else {
+				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
+				prefix = "correct-"
+			}
+			if opts.ExtractPack {
+				err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
+				if err != nil {
+					return err
+				}
+			}
+			if opts.ReuploadBlobs {
+				_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
+				if err != nil {
+					return err
+				}
+				printer.S("         uploaded %v %v", blob.Type, id)
+			}
+		}
+		return nil
+	})
+	return err
+}
+
+func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
+	filename := fmt.Sprintf("%s%s.bin", prefix, id)
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write(plain)
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	printer.S("decrypt of blob %v stored at %v", id, filename)
+	return nil
+}

internal/repository/index/master_index_test.go

@@ -694,3 +694,62 @@ func TestRewriteSplitPacks(t *testing.T) {
 	blobs := mi.Lookup(blobOther.BlobHandle)
 	rtest.Equals(t, nil, blobs)
 }
+
+// TestRewriteFullPacks checks that Rewrite drops a duplicate full index for the same
+// pack while keeping the other index files and blob lookups intact. Creates 3 indexes:
+// - indexA: contains packA
+// - indexB: contains packB
+// - indexC: contains packB
+// After the rewrite, indexC must be dropped. The other indexes must be kept.
+func TestRewriteFullPacks(t *testing.T) {
+	originalFull := index.Full
+	defer func() {
+		index.Full = originalFull
+	}()
+	index.Full = func(*index.Index) bool { return true }
+
+	repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion)
+
+	packA := restic.NewRandomID()
+	packB := restic.NewRandomID()
+
+	blobA := restic.PackedBlob{
+		PackID: packA,
+		Blob: restic.Blob{
+			BlobHandle: restic.NewRandomBlobHandle(),
+			Length:     uint(crypto.CiphertextLength(10)),
+			Offset:     0,
+		},
+	}
+	blobB := restic.PackedBlob{
+		PackID: packB,
+		Blob: restic.Blob{
+			BlobHandle: restic.NewRandomBlobHandle(),
+			Length:     uint(crypto.CiphertextLength(50)),
+			Offset:     0,
+		},
+	}
+
+	mi := index.NewMasterIndex()
+	rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
+	rtest.OK(t, mi.Flush(context.TODO(), unpacked))
+	rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
+	rtest.OK(t, mi.Flush(context.TODO(), unpacked))
+	rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
+	rtest.OK(t, mi.Flush(context.TODO(), unpacked))
+
+	indexIDs := mi.IDs()
+	rtest.Equals(t, 3, len(indexIDs))
+
+	rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
+
+	mi2 := index.NewMasterIndex()
+	rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
+
+	afterRewrite := mi2.IDs()
+	rtest.Equals(t, 2, len(afterRewrite))
+	rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
+
+	rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
+	rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
+}

internal/repository/index_list.go

@@ -0,0 +1,41 @@
+package repository
+
+import (
+	"context"
+	"iter"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository/index"
+	"github.com/restic/restic/internal/restic"
+)
+
+// IndexBlob is one blob handle from an on-disk index file, or an error from loading/decoding
+// that file.
+type IndexBlob struct {
+	Handle restic.BlobHandle
+	Error  error
+}
+
+// AllIndexBlobs streams blob handles from each index file without building a master index.
+func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.LoaderUnpacked) iter.Seq[IndexBlob] {
+	return func(yield func(IndexBlob) bool) {
+		stopIteration := errors.New("stop index blob iteration")
+		err := index.ForAllIndexes(ctx, lister, loader, func(_ restic.ID, idx *index.Index, err error) error {
+			if err != nil {
+				return err
+			}
+			for blob := range idx.Values() {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+				if !yield(IndexBlob{Handle: blob.BlobHandle}) {
+					return stopIteration
+				}
+			}
+			return nil
+		})
+		if err != nil && !errors.Is(err, stopIteration) {
+			yield(IndexBlob{Error: err})
+		}
+	}
+}

internal/repository/index_list_test.go

@@ -0,0 +1,62 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestAllIndexBlobs(t *testing.T) {
+	repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
+
+	want := restic.NewBlobSet()
+	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
+		for i := range 5 {
+			data := []byte{byte('a' + i)}
+			id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, data, restic.ID{}, false)
+			rtest.OK(t, err)
+			want.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id})
+		}
+		return nil
+	}))
+
+	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
+
+	fromMaster := restic.NewBlobSet()
+	rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
+		fromMaster.Insert(pb.BlobHandle)
+	}))
+	rtest.Equals(t, want, fromMaster)
+
+	fromStream := restic.NewBlobSet()
+	for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
+		if entry.Error != nil {
+			t.Fatalf("unexpected error: %v", entry.Error)
+		}
+		fromStream.Insert(entry.Handle)
+	}
+	rtest.Equals(t, want, fromStream)
+}
+
+func TestAllIndexBlobsEarlyStop(t *testing.T) {
+	repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
+
+	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
+		for range 5 {
+			_, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, []byte("test"), restic.ID{}, false)
+			rtest.OK(t, err)
+		}
+		return nil
+	}))
+
+	var count int
+	for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
+		rtest.Assert(t, entry.Error == nil, "unexpected error after early stop: %v", entry.Error)
+		count++
+		break
+	}
+	rtest.Equals(t, 1, count)
+}

internal/repository/repack_test.go

@@ -86,7 +86,7 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
 	blobs := restic.NewBlobSet()
 
 	err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
-		entries, _, err := repo.ListPack(context.TODO(), id, size)
+		entries, err := repo.ListPack(context.TODO(), id, size)
 		if err != nil {
 			t.Fatalf("error listing pack %v: %v", id, err)
 		}

internal/repository/repair_pack.go

@@ -76,7 +76,7 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
 
 	err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		if ids.Has(id) {
-			blobs, _, err := repo.ListPack(ctx, id, size)
+			blobs, err := repo.ListPack(ctx, id, size)
 			if err != nil {
 				return nil
 			}

internal/repository/repository.go

@@ -380,11 +380,13 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 
 	uncompressedLength := 0
 	if r.cfg.Version > 1 {
-
 		// we have a repo v2, so compression is available. if the user opts to
 		// not compress, we won't compress any data, but everything else is
 		// compressed.
-		if r.opts.Compression != CompressionOff || t != restic.DataBlob {
+		// uncompressedLength != 0 is used to indicate compressed data. Thus, a zero-sized blob
+		// cannot be compressed. This special case is only relevant for tests, normal operation does not
+		// generate zero-sized blobs.
+		if len(data) > 0 && (r.opts.Compression != CompressionOff || t != restic.DataBlob) {
 			uncompressedLength = len(data)
 			data = r.getZstdEncoder().EncodeAll(data, nil)
 		}
@@ -781,7 +783,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
 	// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
 	worker := func() error {
 		for fi := range ch {
-			entries, _, err := r.ListPack(wgCtx, fi.ID, fi.Size)
+			entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
 			if err != nil {
 				debug.Log("unable to list pack file %v", fi.ID.Str())
 				m.Lock()
@@ -842,8 +844,13 @@ func (r *Repository) prepareCache() error {
 
 	packs := r.idx.Packs(restic.NewIDSet())
 
+	ids := make(map[string]struct{})
+	for id := range packs {
+		ids[id.String()] = struct{}{}
+	}
+
 	// clear old packs
-	return r.cache.Clear(restic.PackFile, packs)
+	return r.cache.Clear(backend.PackFile, ids)
 }
 
 // SearchKey finds a key with the supplied password, afterwards the config is
@@ -957,12 +964,11 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
 	})
 }
 
-// ListPack returns the list of blobs saved in the pack id and the length of
-// the pack header.
-func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, uint32, error) {
+// ListPack returns the list of blobs saved in the pack id.
+func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
 	h := backend.Handle{Type: restic.PackFile, Name: id.String()}
 
-	entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
+	entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
 	if err != nil {
 		if r.cache != nil {
 			// ignore error as there is not much we can do here
@@ -970,9 +976,9 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
 		}
 
 		// retry on error
-		entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
+		entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
 	}
-	return entries, hdrSize, err
+	return entries, err
 }
 
 // Delete calls backend.Delete() if implemented, and returns an error

internal/repository/repository_test.go

@@ -479,7 +479,7 @@ func TestListPack(t *testing.T) {
 		return nil
 	}))
 
-	blobs, _, err := repo.ListPack(context.TODO(), packID, size)
+	blobs, err := repo.ListPack(context.TODO(), packID, size)
 	rtest.OK(t, err)
 	rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
 

internal/restic/backend_find.go

@@ -24,13 +24,13 @@ func (e *NoIDByPrefixError) Error() string {
 // Find loads the list of all files of type t and searches for names which
 // start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
 // If more than one is found, nil and ErrMultipleIDMatches is returned.
-func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
+func Find(ctx context.Context, repo Lister, t FileType, prefix string) (ID, error) {
 	match := ID{}
 
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	err := be.List(ctx, t, func(id ID, _ int64) error {
+	err := repo.List(ctx, t, func(id ID, _ int64) error {
 		name := id.String()
 		if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
 			if match.IsNull() {

internal/restic/blob.go

@@ -57,10 +57,6 @@ func (h BlobHandle) String() string {
 	return fmt.Sprintf("<%s/%s>", h.Type, h.ID.Str())
 }
 
-func NewRandomBlobHandle() BlobHandle {
-	return BlobHandle{ID: NewRandomID(), Type: DataBlob}
-}
-
 // BlobType specifies what a blob stored in a pack is.
 type BlobType uint8
 

internal/restic/config.go

@@ -26,11 +26,6 @@ const MaxRepoVersion = 2
 // is newly created with Init().
 const StableRepoVersion = 2
 
-// JSONUnpackedLoader loads unpacked JSON.
-type JSONUnpackedLoader interface {
-	LoadJSONUnpacked(context.Context, FileType, ID, interface{}) error
-}
-
 // CreateConfig creates a config file with a randomly selected polynomial and
 // ID.
 func CreateConfig(version uint) (Config, error) {

internal/restic/id.go

@@ -1,11 +1,9 @@
 package restic
 
 import (
-	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
-	"io"
 )
 
 // Hash returns the ID for data.
@@ -40,17 +38,6 @@ func (id ID) String() string {
 	return hex.EncodeToString(id[:])
 }
 
-// NewRandomID returns a randomly generated ID. When reading from rand fails,
-// the function panics.
-func NewRandomID() ID {
-	id := ID{}
-	_, err := io.ReadFull(rand.Reader, id[:])
-	if err != nil {
-		panic(err)
-	}
-	return id
-}
-
 const shortStr = 4
 
 // Str returns the shortened string version of id.

internal/restic/repository.go

@@ -32,9 +32,8 @@ type Repository interface {
 	// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
 	ListBlobs(ctx context.Context, fn func(PackedBlob)) error
 	ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
-	// ListPack returns the list of blobs saved in the pack id and the length of
-	// the pack header.
-	ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, hdrSize uint32, err error)
+	// ListPack returns the list of blobs saved in the pack id.
+	ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
 
 	LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
 	LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error

internal/restic/testing.go

@@ -1,7 +1,9 @@
 package restic
 
 import (
+	"crypto/rand"
 	"fmt"
+	"io"
 )
 
 // TestParseID parses s as a ID and panics if that fails.
@@ -18,3 +20,18 @@ func TestParseID(s string) ID {
 func TestParseHandle(s string, t BlobType) BlobHandle {
 	return BlobHandle{ID: TestParseID(s), Type: t}
 }
+
+func NewRandomBlobHandle() BlobHandle {
+	return BlobHandle{ID: NewRandomID(), Type: DataBlob}
+}
+
+// NewRandomID returns a randomly generated ID. When reading from rand fails,
+// the function panics.
+func NewRandomID() ID {
+	id := ID{}
+	_, err := io.ReadFull(rand.Reader, id[:])
+	if err != nil {
+		panic(err)
+	}
+	return id
+}