Set development version for 0.19.1

Polish changelogs

.golangci.yml

@@ -32,6 +32,7 @@ linters:
         backend-imports:
           files:
             - "**/internal/backend/**"
+            - "!**/internal/backend/cache/**"
             - "!**/internal/backend/test/**"
             - "!**/*_test.go"
           deny:
@@ -39,13 +40,6 @@ linters:
               desc: "internal/restic should not be imported to keep the architectural layers intact"
             - pkg: "github.com/restic/restic/internal/repository"
               desc: "internal/repository should not be imported to keep the architectural layers intact"
-        repository-internals:
-          files:
-            - "**"
-            - "!**/internal/repository/**"
-          deny:
-            - pkg: "github.com/restic/restic/internal/repository/"
-              desc: "packages below internal/repository should not be imported to not depend on repository internals"
     importas:
       alias:
         - pkg: github.com/restic/restic/internal/test

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Table of Contents
 
+* [Changelog for 0.19.1](#changelog-for-restic-0191-2026-07-05)
 * [Changelog for 0.19.0](#changelog-for-restic-0190-2026-06-09)
 * [Changelog for 0.18.1](#changelog-for-restic-0181-2025-09-21)
 * [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
@@ -41,6 +42,116 @@
 * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
 
 
+# Changelog for restic 0.19.1 (2026-07-05)
+The following sections list the changes in restic 0.19.1 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #5234: Prevent mounting over the repository directory
+ * Fix #5667: Skip inaccessible `backup` source paths
+ * Fix #5722: Update `mount` latest symlink after snapshot reload
+ * Fix #21866: Hide `stats` progress bar in JSON mode
+ * Fix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
+ * Fix #21876: Show timezone location in `snapshots` output
+ * Fix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
+ * Fix #21895: Remove read-only files via the SFTP backend on Windows servers
+ * Fix #21899: Make `backup` respect excludes for duplicate directory entries
+
+## Details
+
+ * Bugfix #5234: Prevent mounting over the repository directory
+
+   Using a local repository directory as the `mount` target — or a path that
+   contains it, or that it contains — caused the FUSE server to read its own
+   backend files through the new mount, deadlocking the kernel and requiring a long
+   reboot to recover.
+
+   Restic now resolves both paths and refuses any such overlap with a clear error
+   before mounting.
+
+   https://github.com/restic/restic/issues/5234
+   https://github.com/restic/restic/pull/5348
+
+ * Bugfix #5667: Skip inaccessible `backup` source paths
+
+   The `backup` command only skipped source paths that did not exist. A path that
+   could not be accessed for another reason, such as a malformed path on Windows,
+   was kept and produced an empty snapshot. Restic now skips any such path and
+   aborts if none remain.
+
+   https://github.com/restic/restic/issues/5667
+   https://github.com/restic/restic/pull/21852
+
+ * Bugfix #5722: Update `mount` latest symlink after snapshot reload
+
+   When `restic mount` was kept running while new snapshots were created, the new
+   snapshots appeared in the mountpoint, but the `latest` symlink could still point
+   to the previously latest snapshot. Restic now invalidates the cached snapshot
+   directory entries after a snapshot reload so that `latest` points to the newest
+   snapshot.
+
+   https://github.com/restic/restic/issues/5722
+   https://github.com/restic/restic/pull/21873
+
+ * Bugfix #21866: Hide `stats` progress bar in JSON mode
+
+   Since restic 0.19.0, the `stats` command shows a progress bar. This progress bar
+   was unintentionally displayed also when using the `--json` option, mixing
+   regular text output with JSON. This is now fixed.
+
+   https://github.com/restic/restic/issues/21866
+   https://github.com/restic/restic/pull/21871
+
+ * Bugfix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
+
+   Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>` to
+   no longer group snapshots by host and paths by default.
+
+   The `snapshots --latest <n>` command now again uses the old behavior of grouping
+   by host and paths when `--group-by` is not specified. However, when specifying
+   `--group-by` the output is still grouped as requested, as in restic 0.19.0.
+
+   https://github.com/restic/restic/issues/21869
+   https://github.com/restic/restic/pull/21875
+
+ * Bugfix #21876: Show timezone location in `snapshots` output
+
+   With restic 0.19.0, the `snapshots` command printed the current timezone when
+   listing snapshots. However, that timezone label might change during the year,
+   for example with daylight saving time. Restic now prints a more consistent and
+   shorter version of the text.
+
+   https://github.com/restic/restic/pull/21876
+   https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
+
+ * Bugfix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
+
+   Since restic 0.19.0, the `mount` command validates a mountpoint before loading
+   the repository. If restic was unable to stat the mountpoint, this would result
+   in a crash. This has now been fixed to correctly return an error instead.
+
+   https://github.com/restic/restic/pull/21879
+
+ * Bugfix #21895: Remove read-only files via the SFTP backend on Windows servers
+
+   Since restic 0.19.0, repository files on the SFTP backend are marked read-only
+   after save. On Windows SFTP servers, removing them failed with a permission
+   error. The SFTP backend now clears the read-only flag before removing the file.
+
+   https://github.com/restic/restic/issues/21895
+   https://github.com/restic/restic/pull/21897
+
+ * Bugfix #21899: Make `backup` respect excludes for duplicate directory entries
+
+   Since restic 0.19.0, backing up a directory with duplicate directory entries
+   always produced "Warning: at least one source file could not be read", even when
+   those files were excluded. This has now been fixed.
+
+   https://github.com/restic/restic/issues/21899
+   https://github.com/restic/restic/pull/21900
+
+
 # Changelog for restic 0.19.0 (2026-06-09)
 The following sections list the changes in restic 0.19.0 relevant to
 restic users. The changes are ordered by importance.

VERSION

@@ -1 +1 @@
-0.19.0-dev
+0.19.1-dev

changelog/0.19.1_2026-07-05/issue-21866

@@ -0,0 +1,8 @@
+Bugfix: Hide `stats` progress bar in JSON mode
+
+Since restic 0.19.0, the `stats` command shows a progress bar. This
+progress bar was unintentionally displayed also when using the `--json`
+option, mixing regular text output with JSON. This is now fixed.
+
+https://github.com/restic/restic/issues/21866
+https://github.com/restic/restic/pull/21871

changelog/0.19.1_2026-07-05/issue-21869

@@ -0,0 +1,12 @@
+Bugfix: Restore old behavior of `snapshots --latest <n>` without `--group-by`
+
+Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>`
+to no longer group snapshots by host and paths by default.
+
+The `snapshots --latest <n>` command now again uses the old behavior of
+grouping by host and paths when `--group-by` is not specified. However, when
+specifying `--group-by` the output is still grouped as requested, as in restic
+0.19.0.
+
+https://github.com/restic/restic/issues/21869
+https://github.com/restic/restic/pull/21875

changelog/0.19.1_2026-07-05/issue-21895

@@ -0,0 +1,9 @@
+Bugfix: Remove read-only files via the SFTP backend on Windows servers
+
+Since restic 0.19.0, repository files on the SFTP backend are marked
+read-only after save. On Windows SFTP servers, removing them failed
+with a permission error. The SFTP backend now clears the read-only flag
+before removing the file.
+
+https://github.com/restic/restic/issues/21895
+https://github.com/restic/restic/pull/21897

changelog/0.19.1_2026-07-05/issue-21899

@@ -0,0 +1,8 @@
+Bugfix: Make `backup` respect excludes for duplicate directory entries
+
+Since restic 0.19.0, backing up a directory with duplicate directory
+entries always produced "Warning: at least one source file could not be
+read", even when those files were excluded. This has now been fixed.
+
+https://github.com/restic/restic/issues/21899
+https://github.com/restic/restic/pull/21900

changelog/0.19.1_2026-07-05/issue-5667

@@ -0,0 +1,9 @@
+Bugfix: Skip inaccessible `backup` source paths
+
+The `backup` command only skipped source paths that did not exist. A path that
+could not be accessed for another reason, such as a malformed path on Windows,
+was kept and produced an empty snapshot. Restic now skips any such path and
+aborts if none remain.
+
+https://github.com/restic/restic/issues/5667
+https://github.com/restic/restic/pull/21852

changelog/0.19.1_2026-07-05/issue-5722

@@ -0,0 +1,10 @@
+Bugfix: Update `mount` latest symlink after snapshot reload
+
+When `restic mount` was kept running while new snapshots were
+created, the new snapshots appeared in the mountpoint, but the `latest`
+symlink could still point to the previously latest snapshot. Restic now
+invalidates the cached snapshot directory entries after a snapshot reload so
+that `latest` points to the newest snapshot.
+
+https://github.com/restic/restic/issues/5722
+https://github.com/restic/restic/pull/21873

changelog/0.19.1_2026-07-05/pull-21876

@@ -0,0 +1,9 @@
+Bugfix: Show timezone location in `snapshots` output
+
+With restic 0.19.0, the `snapshots` command printed the current timezone when
+listing snapshots. However, that timezone label might change during the year,
+for example with daylight saving time. Restic now prints a more consistent and
+shorter version of the text.
+
+https://github.com/restic/restic/pull/21876
+https://forum.restic.net/t/possible-bug-in-timezone-naming/10867

changelog/0.19.1_2026-07-05/pull-21879

@@ -0,0 +1,7 @@
+Bugfix: Prevent crash in mountpoint validation if mountpoint is inaccessible
+
+Since restic 0.19.0, the `mount` command validates a mountpoint before loading
+the repository. If restic was unable to stat the mountpoint, this would result
+in a crash. This has now been fixed to correctly return an error instead.
+
+https://github.com/restic/restic/pull/21879

cmd/restic/cmd_backup.go

@@ -171,13 +171,17 @@ var ErrInvalidSourceData = errors.New("at least one source file could not be rea
 // ErrNoSourceData is used to report that no source data was found
 var ErrNoSourceData = errors.Fatal("all source directories/files do not exist")
 
-// filterExisting returns a slice of all existing items, or an error if no
+// filterExisting returns the items that exist and can be accessed. It returns
-// items exist at all.
+// ErrNoSourceData if none remain, or ErrInvalidSourceData if some were skipped.
 func filterExisting(items []string, warnf func(msg string, args ...interface{})) (result []string, err error) {
 	for _, item := range items {
 		_, err := fs.Lstat(item)
-		if errors.Is(err, os.ErrNotExist) {
+		if err != nil {
-			warnf("%v does not exist, skipping\n", item)
+			if errors.Is(err, os.ErrNotExist) {
+				warnf("%v does not exist, skipping\n", item)
+			} else {
+				warnf("%v cannot be accessed, skipping\n", item)
+			}
 			continue
 		}
 

cmd/restic/cmd_backup_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/errors"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -76,6 +77,28 @@ func TestCollectTargets(t *testing.T) {
 	rtest.Assert(t, err == ErrInvalidSourceData, "expected error when not all targets exist")
 }
 
+func TestFilterExistingUnreadable(t *testing.T) {
+	dir := rtest.TempDir(t)
+
+	existing := filepath.Join(dir, "existing")
+	rtest.OK(t, os.Mkdir(existing, 0755))
+
+	file := filepath.Join(dir, "file")
+	rtest.OK(t, os.WriteFile(file, []byte("x"), 0600))
+
+	// Regression test for #5667. A target whose Lstat fails with an error other
+	// than ErrNotExist must be skipped (ENOTDIR on unix, NUL byte everywhere).
+	for _, unreadable := range []string{filepath.Join(file, "child"), "invalid\x00path"} {
+		result, err := filterExisting([]string{unreadable}, t.Logf)
+		rtest.Assert(t, errors.Is(err, ErrNoSourceData), "input %q: expected ErrNoSourceData; got %v", unreadable, err)
+		rtest.Assert(t, len(result) == 0, "input %q: expected no targets; got %v", unreadable, result)
+
+		result, err = filterExisting([]string{existing, unreadable}, t.Logf)
+		rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "input %q: expected ErrInvalidSourceData; got %v", unreadable, err)
+		rtest.Equals(t, []string{existing}, result)
+	}
+}
+
 func TestReadFilenamesRaw(t *testing.T) {
 	// These should all be returned exactly as-is.
 	expected := []string{

cmd/restic/cmd_check.go

@@ -307,7 +307,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
 	go chkr.Packs(ctx, errChan)
 
 	for err := range errChan {
-		var packErr *repository.ErrPackMetadata
+		var packErr *repository.PackError
 		if errors.As(err, &packErr) {
 			if packErr.Orphaned {
 				orphanedPacks++

cmd/restic/cmd_copy_integration_test.go

@@ -103,7 +103,7 @@ func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks i
 		defer unlock()
 
 		rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
-			blobs, err := repo.ListPack(context.TODO(), id, size)
+			blobs, _, err := repo.ListPack(context.TODO(), id, size)
 			rtest.OK(t, err)
 			rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
 

cmd/restic/cmd_debug.go

@@ -4,19 +4,31 @@ package main
 
 import (
 	"context"
+	"crypto/aes"
+	"crypto/cipher"
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"runtime"
+	"sync"
+	"time"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+	"golang.org/x/sync/errgroup"
 
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/repository/index"
+	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/progress"
 )
 
 func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) {
@@ -116,6 +128,61 @@ func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io
 	})
 }
 
+// Pack is the struct used in printPacks.
+type Pack struct {
+	Name string `json:"name"`
+
+	Blobs []Blob `json:"blobs"`
+}
+
+// Blob is the struct used in printPacks.
+type Blob struct {
+	Type   restic.BlobType `json:"type"`
+	Length uint            `json:"length"`
+	ID     restic.ID       `json:"id"`
+	Offset uint            `json:"offset"`
+}
+
+func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error {
+
+	var m sync.Mutex
+	return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+		blobs, _, err := repo.ListPack(ctx, id, size)
+		if err != nil {
+			printer.E("error for pack %v: %v", id.Str(), err)
+			return nil
+		}
+
+		p := Pack{
+			Name:  id.String(),
+			Blobs: make([]Blob, len(blobs)),
+		}
+		for i, blob := range blobs {
+			p.Blobs[i] = Blob{
+				Type:   blob.Type,
+				Length: blob.Length,
+				ID:     blob.ID,
+				Offset: blob.Offset,
+			}
+		}
+
+		m.Lock()
+		defer m.Unlock()
+		return prettyPrintJSON(wr, p)
+	})
+}
+
+func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
+	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
+		printer.S("index_id: %v", id)
+		if err != nil {
+			return err
+		}
+
+		return idx.Dump(wr)
+	})
+}
+
 func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
 	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
 
@@ -133,11 +200,11 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
 
 	switch tpe {
 	case "indexes":
-		return repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
+		return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
 	case "snapshots":
 		return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
 	case "packs":
-		return repository.DumpPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
+		return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
 	case "all":
 		printer.S("snapshots:")
 		err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
@@ -146,7 +213,7 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
 		}
 
 		printer.S("indexes:")
-		err = repository.DumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
+		err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
 		if err != nil {
 			return err
 		}
@@ -157,6 +224,225 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
 	}
 }
 
+func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
+	if bytewise {
+		printer.S("        trying to repair blob by finding a broken byte")
+	} else {
+		printer.S("        trying to repair blob with single bit flip")
+	}
+
+	ch := make(chan int)
+	var wg errgroup.Group
+	done := make(chan struct{})
+	var fixed []byte
+	var found bool
+
+	workers := runtime.GOMAXPROCS(0)
+	printer.S("         spinning up %d worker functions", runtime.GOMAXPROCS(0))
+	for i := 0; i < workers; i++ {
+		wg.Go(func() error {
+			// make a local copy of the buffer
+			buf := make([]byte, len(input))
+			copy(buf, input)
+
+			testFlip := func(idx int, pattern byte) bool {
+				// flip bits
+				buf[idx] ^= pattern
+
+				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
+				if err == nil {
+					printer.S("")
+					printer.S("        blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
+					printer.S("        hash is %v", restic.Hash(plaintext))
+					close(done)
+					found = true
+					fixed = plaintext
+					return true
+				}
+
+				// flip bits back
+				buf[idx] ^= pattern
+				return false
+			}
+
+			for i := range ch {
+				if bytewise {
+					for j := 0; j < 255; j++ {
+						if testFlip(i, byte(j)) {
+							return nil
+						}
+					}
+				} else {
+					for j := 0; j < 7; j++ {
+						// flip each bit once
+						if testFlip(i, (1 << uint(j))) {
+							return nil
+						}
+					}
+				}
+			}
+			return nil
+		})
+	}
+
+	wg.Go(func() error {
+		defer close(ch)
+
+		start := time.Now()
+		info := time.Now()
+		for i := range input {
+			select {
+			case ch <- i:
+			case <-done:
+				printer.S("     done after %v", time.Since(start))
+				return nil
+			}
+
+			if time.Since(info) > time.Second {
+				secs := time.Since(start).Seconds()
+				gps := float64(i) / secs
+				remaining := len(input) - i
+				eta := time.Duration(float64(remaining)/gps) * time.Second
+
+				printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
+					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
+				info = time.Now()
+			}
+		}
+		return nil
+	})
+	err := wg.Wait()
+	if err != nil {
+		panic("all go routines can only return nil")
+	}
+
+	if !found {
+		printer.S("\n        blob could not be repaired")
+	}
+	return fixed
+}
+
+func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
+	// strip signature at the end
+	l := len(buf)
+	nonce, ct := buf[:16], buf[16:l-16]
+	out := make([]byte, len(ct))
+
+	c, err := aes.NewCipher(k.EncryptionKey[:])
+	if err != nil {
+		panic(fmt.Sprintf("unable to create cipher: %v", err))
+	}
+	e := cipher.NewCTR(c, nonce)
+	e.XORKeyStream(out, ct)
+
+	return out
+}
+
+func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
+	dec, err := zstd.NewReader(nil)
+	if err != nil {
+		panic(err)
+	}
+
+	pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
+	// allow processing broken pack files
+	if pack == nil {
+		return err
+	}
+
+	err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
+		for _, blob := range list {
+			printer.S("      loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
+			if int(blob.Offset+blob.Length) > len(pack) {
+				printer.E("skipping truncated blob")
+				continue
+			}
+			buf := pack[blob.Offset : blob.Offset+blob.Length]
+			key := repo.Key()
+
+			nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+			plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+			outputPrefix := ""
+			filePrefix := ""
+			if err != nil {
+				printer.E("error decrypting blob: %v", err)
+				if opts.TryRepair || opts.RepairByte {
+					plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
+				}
+				if plaintext != nil {
+					outputPrefix = "repaired "
+					filePrefix = "repaired-"
+				} else {
+					plaintext = decryptUnsigned(key, buf)
+					err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
+					if err != nil {
+						return err
+					}
+					continue
+				}
+			}
+
+			if blob.IsCompressed() {
+				decompressed, err := dec.DecodeAll(plaintext, nil)
+				if err != nil {
+					printer.S("         failed to decompress blob %v", blob.ID)
+				}
+				if decompressed != nil {
+					plaintext = decompressed
+				}
+			}
+
+			id := restic.Hash(plaintext)
+			var prefix string
+			if !id.Equal(blob.ID) {
+				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
+				prefix = "wrong-hash-"
+			} else {
+				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
+				prefix = "correct-"
+			}
+			if opts.ExtractPack {
+				err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
+				if err != nil {
+					return err
+				}
+			}
+			if opts.ReuploadBlobs {
+				_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
+				if err != nil {
+					return err
+				}
+				printer.S("         uploaded %v %v", blob.Type, id)
+			}
+		}
+		return nil
+	})
+	return err
+}
+
+func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
+	filename := fmt.Sprintf("%s%s.bin", prefix, id)
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write(plain)
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	printer.S("decrypt of blob %v stored at %v", id, filename)
+	return nil
+}
+
 func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
 	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
 
@@ -192,14 +478,8 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
 		return err
 	}
 
-	examineOpts := repository.ExaminePackOptions{
-		TryRepair:     opts.TryRepair,
-		RepairByte:    opts.RepairByte,
-		ExtractPack:   opts.ExtractPack,
-		ReuploadBlobs: opts.ReuploadBlobs,
-	}
 	for _, id := range ids {
-		err := repository.ExaminePack(ctx, repo, id, examineOpts, printer)
+		err := examinePack(ctx, opts, repo, id, printer)
 		if err != nil {
 			printer.E("error: %v", err)
 		}
@@ -209,3 +489,78 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
 	}
 	return nil
 }
+
+func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
+	printer.S("examine %v", id)
+
+	buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
+	// also process damaged pack files
+	if buf == nil {
+		return err
+	}
+	printer.S("  file size is %v", len(buf))
+	gotID := restic.Hash(buf)
+	if !id.Equal(gotID) {
+		printer.S("  wanted hash %v, got %v", id, gotID)
+	} else {
+		printer.S("  hash for file content matches")
+	}
+
+	printer.S("  ========================================")
+	printer.S("  looking for info in the indexes")
+
+	blobsLoaded := false
+	// examine all data the indexes have for the pack file
+	for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
+		blobs := b.Blobs
+		if len(blobs) == 0 {
+			continue
+		}
+
+		checkPackSize(blobs, len(buf), printer)
+
+		err = loadBlobs(ctx, opts, repo, id, blobs, printer)
+		if err != nil {
+			printer.E("error: %v", err)
+		} else {
+			blobsLoaded = true
+		}
+	}
+
+	printer.S("  ========================================")
+	printer.S("  inspect the pack itself")
+
+	blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
+	if err != nil {
+		return fmt.Errorf("pack %v: %v", id.Str(), err)
+	}
+	checkPackSize(blobs, len(buf), printer)
+
+	if !blobsLoaded {
+		return loadBlobs(ctx, opts, repo, id, blobs, printer)
+	}
+	return nil
+}
+
+func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
+	// track current size and offset
+	var size, offset uint64
+
+	blobs.Sort()
+
+	for _, pb := range blobs {
+		printer.S("      %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
+		if offset != uint64(pb.Offset) {
+			printer.S("      hole in file, want offset %v, got %v", offset, pb.Offset)
+		}
+		offset = uint64(pb.Offset + pb.Length)
+		size += uint64(pb.Length)
+	}
+	size += uint64(pack.CalculateHeaderSize(blobs))
+
+	if uint64(fileSize) != size {
+		printer.S("      file sizes do not match: computed %v, file size is %v", size, fileSize)
+	} else {
+		printer.S("      file sizes match")
+	}
+}

cmd/restic/cmd_find.go

@@ -472,7 +472,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 			delete(packIDs, idStr)
 		}
 		debug.Log("Found pack %s", idStr)
-		blobs, err := f.repo.ListPack(ctx, id, size)
+		blobs, _, err := f.repo.ListPack(ctx, id, size)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_forget.go

@@ -261,7 +261,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 			}
 
 			var key data.SnapshotGroupKey
-			if json.Unmarshal([]byte(k), &key) != nil {
+			if err := json.Unmarshal([]byte(k), &key); err != nil {
 				return err
 			}
 

cmd/restic/cmd_list.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
-	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
 
@@ -69,13 +69,18 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		for entry := range repository.AllIndexBlobs(ctx, repo, repo) {
+		return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, err error) error {
-			if entry.Error != nil {
+			if err != nil {
-				return entry.Error
+				return err
 			}
-			printer.S("%v %v", entry.Handle.Type, entry.Handle.ID)
+			for blobs := range idx.Values() {
-		}
+				if ctx.Err() != nil {
-		return nil
+					return ctx.Err()
+				}
+				printer.S("%v %v", blobs.Type, blobs.ID)
+			}
+			return nil
+		})
 	default:
 		return errors.Fatal("invalid type")
 	}

cmd/restic/cmd_mount.go

@@ -226,6 +226,8 @@ func validateMountpoint(mountpoint string, gopts global.Options) error {
 	stat, err := os.Stat(mountpoint)
 	if errors.Is(err, os.ErrNotExist) {
 		return errors.Fatal(fmt.Sprintf("mountpoint %s does not exist", mountpoint))
+	} else if err != nil {
+		return errors.Fatal(fmt.Sprintf("mountpoint %s is inaccessible: %v", mountpoint, err))
 	} else if !stat.IsDir() {
 		return errors.Fatal(fmt.Sprintf("mountpoint %s is not a directory", mountpoint))
 	}

cmd/restic/cmd_repair_index_integration_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
+	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -60,6 +61,15 @@ func TestRebuildIndex(t *testing.T) {
 	testRebuildIndex(t, nil)
 }
 
+func TestRebuildIndexAlwaysFull(t *testing.T) {
+	indexFull := index.Full
+	defer func() {
+		index.Full = indexFull
+	}()
+	index.Full = func(*index.Index) bool { return true }
+	testRebuildIndex(t, nil)
+}
+
 // indexErrorBackend modifies the first index after reading.
 type indexErrorBackend struct {
 	backend.Backend

cmd/restic/cmd_snapshots.go

@@ -38,6 +38,9 @@ Exit status is 12 if the password is incorrect.
 `,
 		GroupID:           cmdGroupDefault,
 		DisableAutoGenTag: true,
+		PreRunE: func(_ *cobra.Command, _ []string) error {
+			return opts.Finalize()
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			finalizeSnapshotFilter(&opts.SnapshotFilter)
 			return runSnapshots(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
@@ -52,7 +55,7 @@ Exit status is 12 if the password is incorrect.
 type SnapshotOptions struct {
 	data.SnapshotFilter
 	Compact bool
-	Last    bool // This option should be removed in favour of Latest.
+	last    bool // Deprecated in favour of Latest.
 	Latest  int
 	GroupBy data.SnapshotGroupByOptions
 }
@@ -60,7 +63,7 @@ type SnapshotOptions struct {
 func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
 	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
 	f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format")
-	f.BoolVar(&opts.Last, "last", false, "only show the last snapshot for each host and path")
+	f.BoolVar(&opts.last, "last", false, "only show the last snapshot for each host and path")
 	err := f.MarkDeprecated("last", "use --latest 1")
 	if err != nil {
 		// MarkDeprecated only returns an error when the flag is not found
@@ -70,6 +73,13 @@ func (opts *SnapshotOptions) AddFlags(f *pflag.FlagSet) {
 	f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma")
 }
 
+func (opts *SnapshotOptions) Finalize() error {
+	if opts.last && opts.Latest == 0 {
+		opts.Latest = 1
+	}
+	return nil
+}
+
 func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Options, args []string, term ui.Terminal) error {
 	printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
 	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
@@ -95,12 +105,12 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
 			return ctx.Err()
 		}
 
-		if opts.Last {
+		if opts.Latest > 0 {
-			// This branch should be removed in the same time
+			if grouped {
-			// that --last.
+				list = filterLatestSnapshotsInGroup(list, opts.Latest)
-			list = filterLatestSnapshotsInGroup(list, 1)
+			} else {
-		} else if opts.Latest > 0 {
+				list = filterLatestSnapshots(list, opts.Latest)
-			list = filterLatestSnapshotsInGroup(list, opts.Latest)
+			}
 		}
 		sort.Sort(sort.Reverse(list))
 		snapshotGroups[k] = list
@@ -134,6 +144,43 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts global.Option
 	return nil
 }
 
+// filterLastSnapshotsKey is used by FilterLastSnapshots.
+type filterLastSnapshotsKey struct {
+	Hostname    string
+	JoinedPaths string
+}
+
+// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
+func newFilterLastSnapshotsKey(sn *data.Snapshot) filterLastSnapshotsKey {
+	// Shallow slice copy
+	var paths = make([]string, len(sn.Paths))
+	copy(paths, sn.Paths)
+	sort.Strings(paths)
+	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
+}
+
+// filterLatestSnapshots filters a list of snapshots to only return
+// the limit last entries for each hostname and path. If the snapshot
+// contains multiple paths, they will be joined and treated as one
+// item.
+func filterLatestSnapshots(list data.Snapshots, limit int) data.Snapshots {
+	// Sort the snapshots so that the newer ones are listed first
+	sort.SliceStable(list, func(i, j int) bool {
+		return list[i].Time.After(list[j].Time)
+	})
+
+	var results data.Snapshots
+	seen := make(map[filterLastSnapshotsKey]int)
+	for _, sn := range list {
+		key := newFilterLastSnapshotsKey(sn)
+		if seen[key] < limit {
+			seen[key]++
+			results = append(results, sn)
+		}
+	}
+	return results
+}
+
 // filterLatestSnapshotsInGroup filters a list of snapshots to only return
 // the `limit` last entries. It is assumed that the snapshot list only contains
 // one group of snapshots.
@@ -245,11 +292,11 @@ func PrintSnapshots(stdout io.Writer, list data.Snapshots, reasons []data.KeepRe
 	// Each snapshot can be registered in different timezones,
 	// but we display them all in local timezone on this output.
 	footer := fmt.Sprintf("%d snapshots", len(list))
-	zoneName, _ := time.Now().Local().Zone()
+	zoneName := time.Now().Local().Location().String()
-	if zoneName != "" {
+	if zoneName == "Local" {
-		footer = fmt.Sprintf("Timestamps shown in %s timezone\n%s", zoneName, footer)
+		zoneName = "local time"
 	}
-	tab.AddFooter(footer)
+	tab.AddFooter(fmt.Sprintf("Timestamps shown in %s\n%s", zoneName, footer))
 
 	if multiline {
 		// print an additional blank line between snapshots

cmd/restic/cmd_snapshots_integration_test.go

@@ -35,10 +35,7 @@ func testRunSnapshots(t testing.TB, gopts global.Options) (newest *Snapshot, sna
 	return
 }
 
-func TestSnapshotsGroupByAndLatest(t *testing.T) {
+func snapshotsGroupTestData(t *testing.T, env *testEnvironment, keepPath bool) string {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
 	testSetupBackupData(t, env)
 	// two backups on the same host but with different paths
 	opts := BackupOptions{Host: "testhost", TimeStamp: time.Now().Format(time.DateTime)}
@@ -46,9 +43,22 @@ func TestSnapshotsGroupByAndLatest(t *testing.T) {
 	// Use later timestamp for second backup
 	opts.TimeStamp = time.Now().Add(time.Second).Format(time.DateTime)
 	snapshotsIDs := loadSnapshotMap(t, env.gopts)
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata/0"}, opts, env.gopts)
+
+	targets := []string{"testdata/0"}
+	if keepPath {
+		targets = []string{"testdata"}
+	}
+	testRunBackup(t, filepath.Dir(env.testdata), targets, opts, env.gopts)
 	_, secondSnapshotID := lastSnapshot(snapshotsIDs, loadSnapshotMap(t, env.gopts))
 
+	return secondSnapshotID
+}
+
+func TestSnapshotsGroupByAndLatest(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	secondSnapshotID := snapshotsGroupTestData(t, env, false)
 	buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
 		gopts.JSON = true
 		// only group by host but not path
@@ -65,3 +75,21 @@ func TestSnapshotsGroupByAndLatest(t *testing.T) {
 	rtest.Assert(t, len(snapshots[0].Snapshots) == 1, "expected only one latest snapshot, got %d", len(snapshots[0].Snapshots))
 	rtest.Equals(t, snapshots[0].Snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
 }
+
+func TestSnapshotsLatest(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	secondSnapshotID := snapshotsGroupTestData(t, env, true)
+
+	buf, err := withCaptureStdout(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
+		gopts.JSON = true
+		opts := SnapshotOptions{Latest: 1}
+		return runSnapshots(ctx, opts, gopts, []string{}, gopts.Term)
+	})
+	rtest.OK(t, err)
+	snapshots := []Snapshot{}
+	rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
+	rtest.Assert(t, len(snapshots) == 1, "expected only one snapshot, got %d", len(snapshots))
+	rtest.Equals(t, snapshots[0].ID.String(), secondSnapshotID, "unexpected snapshot ID")
+}

cmd/restic/cmd_stats.go

@@ -141,7 +141,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts global.Options, args
 		snapshots = append(snapshots, sn)
 	}
 
-	statsProgress := newStatsProgress(term, uint64(len(snapshots)))
+	statsProgress := newStatsProgress(term, !gopts.JSON, uint64(len(snapshots)))
 
 	updater := progress.NewUpdater(ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()), func(runtime time.Duration, final bool) {
 		statsProgress.printProgress(runtime, final)
@@ -375,6 +375,7 @@ type statsProgress struct {
 	term          ui.Terminal
 	m             sync.Mutex
 	snapshotCount uint64
+	show          bool
 
 	processedSnapshotCount uint64
 	processedFileCount     uint64
@@ -382,14 +383,18 @@ type statsProgress struct {
 	processedSize          uint64
 }
 
-func newStatsProgress(term ui.Terminal, snapshotCount uint64) *statsProgress {
+func newStatsProgress(term ui.Terminal, show bool, snapshotCount uint64) *statsProgress {
 	return &statsProgress{
 		term:          term,
+		show:          show,
 		snapshotCount: snapshotCount,
 	}
 }
 
 func (s *statsProgress) printProgress(runtime time.Duration, final bool) {
+	if !s.show {
+		return
+	}
 	s.m.Lock()
 
 	progressBase := s.processedSnapshotCount

cmd/restic/cmd_stats_test.go

@@ -66,7 +66,7 @@ func TestSizeHistogramString(t *testing.T) {
 func TestStatsProgress(t *testing.T) {
 	term := &ui.MockTerminal{}
 
-	progress := newStatsProgress(term, 2)
+	progress := newStatsProgress(term, true, 2)
 	progress.printProgress(0*time.Second, false)
 	rtest.Equals(t, []string{"[0:00] 0.00%  0 / 2 snapshots, 0 B"}, term.Output)
 
@@ -88,3 +88,12 @@ func TestStatsProgress(t *testing.T) {
 	progress.printProgress(20*time.Second, true)
 	rtest.Equals(t, []string{"[0:20] 100.00%  2 / 2 snapshots, 4 files, 5 blobs, 6 B"}, term.Output)
 }
+
+func TestStatsProgressJSON(t *testing.T) {
+	term := &ui.MockTerminal{}
+
+	progress := newStatsProgress(term, false, 2)
+	progress.printProgress(0*time.Second, false)
+	// JSON output is not available yet, so just make sure to not break normal json output
+	rtest.Equals(t, nil, term.Output)
+}

internal/archiver/archiver.go

@@ -320,6 +320,8 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 	finder := data.NewTreeFinder(previous)
 	defer finder.Close()
 
+	var lastExcluded string
+
 	for _, name := range names {
 		// test if context has been cancelled
 		if ctx.Err() != nil {
@@ -327,6 +329,12 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 			return futureNode{}, ctx.Err()
 		}
 
+		if name == lastExcluded {
+			// Skip duplicate directory entry if it was already excluded.
+			// This avoids printing errors about duplicate directory entries even though the entry in question is ignored.
+			continue
+		}
+
 		pathname := arch.FS.Join(dir, name)
 		oldNode, err := finder.Find(name)
 		err = arch.error(pathname, err)
@@ -348,6 +356,7 @@ func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, me
 		}
 
 		if excluded {
+			lastExcluded = name
 			continue
 		}
 

internal/archiver/archiver_test.go

@@ -894,6 +894,90 @@ func TestArchiverSaveDir(t *testing.T) {
 	}
 }
 
+type duplicateReaddirFS struct {
+	fs.FS
+	dir   string
+	names []string
+}
+
+func (d *duplicateReaddirFS) OpenFile(name string, flag int, metadataOnly bool) (fs.File, error) {
+	f, err := d.FS.OpenFile(name, flag, metadataOnly)
+	if err != nil {
+		return nil, err
+	}
+
+	if name == d.dir {
+		return &duplicateReaddirFile{File: f, names: d.names}, nil
+	}
+	return f, nil
+}
+
+type duplicateReaddirFile struct {
+	fs.File
+	names []string
+}
+
+func (f *duplicateReaddirFile) Readdirnames(int) ([]string, error) {
+	return append([]string(nil), f.names...), nil
+}
+
+func TestArchiverSaveDirDuplicateExcludedEntry(t *testing.T) {
+	const targetNodeName = "targetdir"
+
+	src := TestDir{
+		"excluded": TestFile{Content: "skip me"},
+		"keep":     TestFile{Content: "keep me"},
+	}
+	tempdir, repo := prepareTempdirRepoSrc(t, src)
+
+	testFS := fs.Track{FS: &duplicateReaddirFS{
+		FS:    &fs.Local{},
+		dir:   ".",
+		names: []string{"excluded", "excluded", "keep"},
+	}}
+	arch := New(repo, testFS, Options{})
+	arch.summary = &Summary{}
+	arch.Select = func(item string, fi *fs.ExtendedFileInfo, _ fs.FS) bool {
+		return filepath.Base(item) != "excluded"
+	}
+	arch.Error = func(item string, err error) error {
+		t.Errorf("unexpected archiver error for %v: %v", item, err)
+		return err
+	}
+
+	back := rtest.Chdir(t, tempdir)
+	defer back()
+
+	// duplicate node check in tree finder is only done if the previous tree is not nil
+	previousTree, err := data.NewTreeNodeIterator(strings.NewReader(`{"nodes":[]}`))
+	rtest.OK(t, err)
+
+	var treeID restic.ID
+	err = repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
+		wg, ctx := errgroup.WithContext(ctx)
+		arch.runWorkers(ctx, wg, uploader)
+		meta, err := testFS.OpenFile(".", fs.O_NOFOLLOW, true)
+		rtest.OK(t, err)
+		ft, err := arch.saveDir(ctx, "/", ".", meta, previousTree, nil)
+		rtest.OK(t, err)
+		rtest.OK(t, meta.Close())
+
+		fnr := ft.take(ctx)
+		node := fnr.node
+		node.Name = targetNodeName
+		treeID = data.TestSaveNodes(t, ctx, uploader, []*data.Node{node})
+		arch.stopWorkers()
+		return wg.Wait()
+	})
+	rtest.OK(t, err)
+
+	TestEnsureTree(context.TODO(), t, "/", repo, treeID, TestDir{
+		"targetdir": TestDir{
+			"keep": TestFile{Content: "keep me"},
+		},
+	})
+}
+
 func TestArchiverSaveDirIncremental(t *testing.T) {
 	tempdir := rtest.TempDir(t)
 

internal/backend/azure/azure.go

@@ -31,9 +31,11 @@ import (
 
 // Backend stores data on an azure endpoint.
 type Backend struct {
-	cfg         Config
+	cfg          Config
-	container   *azContainer.Client
+	container    *azContainer.Client
-	connections uint
+	connections  uint
+	prefix       string
+	listMaxItems int
 	layout.Layout
 
 	accessTier blob.AccessTier
@@ -143,11 +145,12 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	}
 
 	be := &Backend{
-		container:   client,
+		container:    client,
-		cfg:         cfg,
+		cfg:          cfg,
-		connections: cfg.Connections,
+		connections:  cfg.Connections,
-		Layout:      layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
-		accessTier:  accessTier,
+		listMaxItems: defaultListMaxItems,
+		accessTier:   accessTier,
 	}
 
 	return be, nil
@@ -192,6 +195,11 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
 	return be, nil
 }
 
+// SetListMaxItems sets the number of list items to load per request.
+func (be *Backend) SetListMaxItems(i int) {
+	be.listMaxItems = i
+}
+
 // IsNotExist returns true if the error is caused by a not existing file.
 func (be *Backend) IsNotExist(err error) bool {
 	return bloberror.HasCode(err, bloberror.BlobNotFound)
@@ -223,6 +231,11 @@ func (be *Backend) Hasher() hash.Hash {
 	return md5.New()
 }
 
+// Path returns the path in the bucket that is used for this backend.
+func (be *Backend) Path() string {
+	return be.prefix
+}
+
 // useAccessTier determines whether to apply the configured access tier to a given file.
 // For archive access tier, only data files are stored using that class; metadata
 // must remain instantly accessible.
@@ -406,7 +419,7 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
 		prefix += "/"
 	}
 
-	maxI := int32(defaultListMaxItems)
+	maxI := int32(be.listMaxItems)
 
 	opts := &azContainer.ListBlobsFlatOptions{
 		MaxResults: &maxI,

internal/backend/b2/b2.go

@@ -23,9 +23,10 @@ import (
 
 // b2Backend is a backend which stores its data on Backblaze B2.
 type b2Backend struct {
-	client *b2.Client
+	client       *b2.Client
-	bucket *b2.Bucket
+	bucket       *b2.Bucket
-	cfg    Config
+	cfg          Config
+	listMaxItems int
 	layout.Layout
 
 	canDelete bool
@@ -106,11 +107,12 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string,
 	}
 
 	be := &b2Backend{
-		client:    client,
+		client:       client,
-		bucket:    bucket,
+		bucket:       bucket,
-		cfg:       cfg,
+		cfg:          cfg,
-		Layout:    layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
-		canDelete: true,
+		listMaxItems: defaultListMaxItems,
+		canDelete:    true,
 	}
 
 	return be, nil
@@ -138,14 +140,20 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
 	}
 
 	be := &b2Backend{
-		client: client,
+		client:       client,
-		bucket: bucket,
+		bucket:       bucket,
-		cfg:    cfg,
+		cfg:          cfg,
-		Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		listMaxItems: defaultListMaxItems,
 	}
 	return be, nil
 }
 
+// SetListMaxItems sets the number of list items to load per request.
+func (be *b2Backend) SetListMaxItems(i int) {
+	be.listMaxItems = i
+}
+
 func (be *b2Backend) Properties() backend.Properties {
 	return backend.Properties{
 		Connections:      be.cfg.Connections,
@@ -238,6 +246,7 @@ func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewi
 
 	// sanity check
 	if n != rd.Length() {
+		_ = w.Close()
 		return errors.Errorf("wrote %d bytes instead of the expected %d bytes", n, rd.Length())
 	}
 	return errors.Wrap(w.Close(), "Close")
@@ -296,7 +305,7 @@ func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backe
 	defer cancel()
 
 	prefix, _ := be.Basedir(t)
-	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(defaultListMaxItems))
+	iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems))
 
 	for iter.Next() {
 		obj := iter.Object()

internal/backend/cache/backend.go

@@ -7,10 +7,11 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
 )
 
-// cacheBackend wraps a restic.cacheBackend and adds a cache.
+// Backend wraps a restic.Backend and adds a cache.
-type cacheBackend struct {
+type Backend struct {
 	backend.Backend
 	*Cache
 
@@ -23,10 +24,10 @@ type cacheBackend struct {
 }
 
 // ensure Backend implements backend.Backend
-var _ backend.Backend = &cacheBackend{}
+var _ backend.Backend = &Backend{}
 
-func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *cacheBackend {
+func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *Backend {
-	return &cacheBackend{
+	return &Backend{
 		Backend:    be,
 		Cache:      c,
 		inProgress: make(map[backend.Handle]chan struct{}),
@@ -35,7 +36,7 @@ func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface
 }
 
 // Remove deletes a file from the backend and the cache if it has been cached.
-func (b *cacheBackend) Remove(ctx context.Context, h backend.Handle) error {
+func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
 	debug.Log("cache Remove(%v)", h)
 	err := b.Backend.Remove(ctx, h)
 	if err != nil {
@@ -57,7 +58,7 @@ func autoCacheTypes(h backend.Handle) bool {
 }
 
 // Save stores a new file in the backend and the cache.
-func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
 	if !autoCacheTypes(h) {
 		return b.Backend.Save(ctx, h, rd)
 	}
@@ -91,7 +92,7 @@ func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.Re
 	return nil
 }
 
-func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
+func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
 	finish := make(chan struct{})
 
 	b.inProgressMutex.Lock()
@@ -135,7 +136,7 @@ func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
 }
 
 // loadFromCache will try to load the file from the cache.
-func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
+func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
 	rd, inCache, err := b.Cache.load(h, length, offset)
 	if err != nil {
 		return inCache, err
@@ -150,7 +151,7 @@ func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64,
 }
 
 // Load loads a file from the cache or the backend.
-func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
 	b.inProgressMutex.Lock()
 	waitForFinish, inProgress := b.inProgress[h]
 	b.inProgressMutex.Unlock()
@@ -197,7 +198,7 @@ func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, o
 
 // Stat tests whether the backend has a file. If it does not exist but still
 // exists in the cache, it is removed from the cache.
-func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
+func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
 	debug.Log("cache Stat(%v)", h)
 
 	fi, err := b.Backend.Stat(ctx, h)
@@ -210,24 +211,31 @@ func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.File
 }
 
 // IsNotExist returns true if the error is caused by a non-existing file.
-func (b *cacheBackend) IsNotExist(err error) bool {
+func (b *Backend) IsNotExist(err error) bool {
 	return b.Backend.IsNotExist(err)
 }
 
-func (b *cacheBackend) Unwrap() backend.Backend {
+func (b *Backend) Unwrap() backend.Backend {
 	return b.Backend
 }
 
-func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
+func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
 	if !b.Cache.canBeCached(t) {
 		return b.Backend.List(ctx, t, fn)
 	}
 
-	ids := make(map[string]struct{})
+	// will contain the IDs of the files that are in the repository
+	ids := restic.NewIDSet()
 
 	// wrap the original function to also add the file to the ids set
 	wrapFn := func(f backend.FileInfo) error {
-		ids[f.Name] = struct{}{}
+		id, err := restic.ParseID(f.Name)
+		if err != nil {
+			// ignore files with invalid name
+			return nil
+		}
+
+		ids.Insert(id)
 
 		// execute the original function
 		return fn(f)
@@ -252,11 +260,11 @@ func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f b
 }
 
 // Warmup delegates to wrapped backend.
-func (b *cacheBackend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
+func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
 	return b.Backend.Warmup(ctx, h)
 }
 
 // WarmupWait delegates to wrapped backend.
-func (b *cacheBackend) WarmupWait(ctx context.Context, h []backend.Handle) error {
+func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error {
 	return b.Backend.WarmupWait(ctx, h)
 }

internal/backend/cache/cache.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
 )
 
 // Cache manages a local cache.
@@ -42,10 +43,10 @@ func readVersion(dir string) (v uint, err error) {
 
 const cacheVersion = 1
 
-var cacheLayoutPaths = map[backend.FileType]string{
+var cacheLayoutPaths = map[restic.FileType]string{
-	backend.PackFile:     "data",
+	restic.PackFile:     "data",
-	backend.SnapshotFile: "snapshots",
+	restic.SnapshotFile: "snapshots",
-	backend.IndexFile:    "index",
+	restic.IndexFile:    "index",
 }
 
 const cachedirTagSignature = "Signature: 8a477f597d28d172789f06886806bc55\n"

internal/backend/cache/file.go

@@ -12,6 +12,7 @@ import (
 	"github.com/restic/restic/internal/backend/util"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
 )
 
 func (c *Cache) filename(h backend.Handle) string {
@@ -170,7 +171,7 @@ func (c *Cache) remove(h backend.Handle) (bool, error) {
 
 // Clear removes all files of type t from the cache that are not contained in
 // the set valid.
-func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
+func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
 	debug.Log("Clearing cache for %v: %v valid files", t, len(valid))
 	if !c.canBeCached(t) {
 		return nil
@@ -182,12 +183,12 @@ func (c *Cache) Clear(t backend.FileType, valid map[string]struct{}) error {
 	}
 
 	for id := range list {
-		if _, ok := valid[id]; ok {
+		if valid.Has(id) {
 			continue
 		}
 
 		// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
-		if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id})); err != nil && !errors.Is(err, os.ErrNotExist) {
+		if err = os.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
 			return err
 		}
 	}
@@ -200,12 +201,12 @@ func isFile(fi os.FileInfo) bool {
 }
 
 // list returns a list of all files of type T in the cache.
-func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
+func (c *Cache) list(t restic.FileType) (restic.IDSet, error) {
 	if !c.canBeCached(t) {
 		return nil, errors.New("cannot be cached")
 	}
 
-	list := make(map[string]struct{})
+	list := restic.NewIDSet()
 	dir := filepath.Join(c.path, cacheLayoutPaths[t])
 	err := filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
 		if err != nil {
@@ -220,8 +221,12 @@ func (c *Cache) list(t backend.FileType) (map[string]struct{}, error) {
 			return nil
 		}
 
-		id := filepath.Base(name)
+		id, err := restic.ParseID(filepath.Base(name))
-		list[id] = struct{}{}
+		if err != nil {
+			return nil
+		}
+
+		list.Insert(id)
 		return nil
 	})
 

internal/backend/cache/file_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"maps"
 	"math/rand"
 	"os"
 	"runtime"
@@ -19,8 +18,8 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} {
+func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) restic.IDSet {
-	ids := make(map[string]struct{})
+	ids := restic.NewIDSet()
 	for i := 0; i < random.Intn(15)+10; i++ {
 		buf := rtest.Random(random.Int(), 1<<19)
 		id := restic.Hash(buf)
@@ -34,13 +33,13 @@ func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType,
 		if err != nil {
 			t.Fatal(err)
 		}
-		ids[id.String()] = struct{}{}
+		ids.Insert(id)
 	}
 	return ids
 }
 
 // randomID returns a random ID from s.
-func randomID(s map[string]struct{}) string {
+func randomID(s restic.IDSet) restic.ID {
 	for id := range s {
 		return id
 	}
@@ -70,7 +69,7 @@ func load(t testing.TB, c *Cache, h backend.Handle) []byte {
 	return buf
 }
 
-func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{} {
+func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
 	list, err := c.list(tpe)
 	if err != nil {
 		t.Errorf("listing failed: %v", err)
@@ -79,7 +78,7 @@ func listFiles(t testing.TB, c *Cache, tpe backend.FileType) map[string]struct{}
 	return list
 }
 
-func clearFiles(t testing.TB, c *Cache, tpe backend.FileType, valid map[string]struct{}) {
+func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
 	if err := c.Clear(tpe, valid); err != nil {
 		t.Error(err)
 	}
@@ -103,34 +102,34 @@ func TestFiles(t *testing.T) {
 			ids := generateRandomFiles(t, random, tpe, c)
 			id := randomID(ids)
 
-			h := backend.Handle{Type: tpe, Name: id}
+			h := backend.Handle{Type: tpe, Name: id.String()}
 			id2 := restic.Hash(load(t, c, h))
 
-			if id != id2.String() {
+			if !id.Equal(id2) {
-				t.Errorf("wrong data returned, want %v, got %v", id, id2.String())
+				t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
 			}
 
 			if !c.Has(h) {
-				t.Errorf("cache thinks index %v isn't present", id)
+				t.Errorf("cache thinks index %v isn't present", id.Str())
 			}
 
 			list := listFiles(t, c, tpe)
-			if !maps.Equal(ids, list) {
+			if !ids.Equals(list) {
 				t.Errorf("wrong list of index IDs returned, want:\n  %v\ngot:\n  %v", ids, list)
 			}
 
-			clearFiles(t, c, tpe, map[string]struct{}{id: {}})
+			clearFiles(t, c, tpe, restic.NewIDSet(id))
 			list2 := listFiles(t, c, tpe)
-			delete(ids, id)
+			ids.Delete(id)
-			want := map[string]struct{}{id: {}}
+			want := restic.NewIDSet(id)
-			if !maps.Equal(list2, want) {
+			if !list2.Equals(want) {
 				t.Errorf("ClearIndexes removed indexes, want:\n  %v\ngot:\n  %v", list2, want)
 			}
 
-			clearFiles(t, c, tpe, map[string]struct{}{})
+			clearFiles(t, c, tpe, restic.NewIDSet())
-			want = map[string]struct{}{}
+			want = restic.NewIDSet()
 			list3 := listFiles(t, c, tpe)
-			if !maps.Equal(list3, want) {
+			if !list3.Equals(want) {
 				t.Errorf("ClearIndexes returned a wrong list, want:\n  %v\ngot:\n  %v", want, list3)
 			}
 		})

internal/backend/cache/testing.go

@@ -3,17 +3,16 @@ package cache
 import (
 	"testing"
 
+	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
 )
 
-const testCacheID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-
 // TestNewCache returns a cache in a temporary directory which is removed when
 // cleanup is called.
 func TestNewCache(t testing.TB) *Cache {
 	dir := test.TempDir(t)
 	t.Logf("created new cache at %v", dir)
-	cache, err := New(testCacheID, dir)
+	cache, err := New(restic.NewRandomID().String(), dir)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/backend/gs/gs.go

@@ -27,25 +27,27 @@ import (
 	"google.golang.org/api/option"
 )
 
-// gs stores data in a GCS bucket.
+// Backend stores data in a GCS bucket.
 //
 // The service account used to access the bucket must have these permissions:
 //   - storage.objects.create
 //   - storage.objects.delete
 //   - storage.objects.get
 //   - storage.objects.list
-type gs struct {
+type Backend struct {
-	gcsClient   *storage.Client
+	gcsClient    *storage.Client
-	projectID   string
+	projectID    string
-	connections uint
+	connections  uint
-	bucketName  string
+	bucketName   string
-	region      string
+	region       string
-	bucket      *storage.BucketHandle
+	bucket       *storage.BucketHandle
+	prefix       string
+	listMaxItems int
 	layout.Layout
 }
 
 // Ensure that *Backend implements backend.Backend.
-var _ backend.Backend = &gs{}
+var _ backend.Backend = &Backend{}
 
 func NewFactory() location.Factory {
 	return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@@ -84,7 +86,7 @@ func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
 	return gcsClient, nil
 }
 
-func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
+func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
 	_, err := bucket.Attrs(ctx)
 	if err == storage.ErrBucketNotExist {
 		return false, nil
@@ -92,7 +94,9 @@ func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (b
 	return err == nil, err
 }
 
-func open(cfg Config, rt http.RoundTripper) (*gs, error) {
+const defaultListMaxItems = 1000
+
+func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
 	gcsClient, err := getStorageClient(rt)
@@ -100,14 +104,16 @@ func open(cfg Config, rt http.RoundTripper) (*gs, error) {
 		return nil, errors.Wrap(err, "getStorageClient")
 	}
 
-	be := &gs{
+	be := &Backend{
-		gcsClient:   gcsClient,
+		gcsClient:    gcsClient,
-		projectID:   cfg.ProjectID,
+		projectID:    cfg.ProjectID,
-		connections: cfg.Connections,
+		connections:  cfg.Connections,
-		bucketName:  cfg.Bucket,
+		bucketName:   cfg.Bucket,
-		region:      cfg.Region,
+		region:       cfg.Region,
-		bucket:      gcsClient.Bucket(cfg.Bucket),
+		bucket:       gcsClient.Bucket(cfg.Bucket),
-		Layout:      layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		prefix:       cfg.Prefix,
+		Layout:       layout.NewDefaultLayout(cfg.Prefix, path.Join),
+		listMaxItems: defaultListMaxItems,
 	}
 
 	return be, nil
@@ -155,12 +161,17 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
 	return be, nil
 }
 
+// SetListMaxItems sets the number of list items to load per request.
+func (be *Backend) SetListMaxItems(i int) {
+	be.listMaxItems = i
+}
+
 // IsNotExist returns true if the error is caused by a not existing file.
-func (be *gs) IsNotExist(err error) bool {
+func (be *Backend) IsNotExist(err error) bool {
 	return errors.Is(err, storage.ErrObjectNotExist)
 }
 
-func (be *gs) IsPermanentError(err error) bool {
+func (be *Backend) IsPermanentError(err error) bool {
 	if be.IsNotExist(err) {
 		return true
 	}
@@ -175,7 +186,7 @@ func (be *gs) IsPermanentError(err error) bool {
 	return false
 }
 
-func (be *gs) Properties() backend.Properties {
+func (be *Backend) Properties() backend.Properties {
 	return backend.Properties{
 		Connections:      be.connections,
 		HasAtomicReplace: true,
@@ -183,12 +194,17 @@ func (be *gs) Properties() backend.Properties {
 }
 
 // Hasher may return a hash function for calculating a content hash for the backend
-func (be *gs) Hasher() hash.Hash {
+func (be *Backend) Hasher() hash.Hash {
 	return md5.New()
 }
 
+// Path returns the path in the bucket that is used for this backend.
+func (be *Backend) Path() string {
+	return be.prefix
+}
+
 // Save stores data in the backend at the handle.
-func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
 	objName := be.Filename(h)
 
 	// Set chunk size to zero to disable resumable uploads.
@@ -238,14 +254,14 @@ func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReade
 
 // Load runs fn with a reader that yields the contents of the file at h at the
 // given offset.
-func (be *gs) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
 }
 
-func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
+func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
 	if length == 0 {
 		// negative length indicates read till end to GCS lib
 		length = -1
@@ -267,7 +283,7 @@ func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offs
 }
 
 // Stat returns information about a blob.
-func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
+func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
 	objName := be.Filename(h)
 
 	attr, err := be.bucket.Object(objName).Attrs(ctx)
@@ -280,7 +296,7 @@ func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo,
 }
 
 // Remove removes the blob with the given name and type.
-func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
+func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
 	objName := be.Filename(h)
 
 	err := be.bucket.Object(objName).Delete(ctx)
@@ -294,7 +310,7 @@ func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
 
 // List runs fn for each file in the backend which has the type t. When an
 // error occurs (or fn returns an error), List stops and returns it.
-func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
+func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
 	prefix, _ := be.Basedir(t)
 
 	// make sure prefix ends with a slash
@@ -339,15 +355,15 @@ func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.File
 }
 
 // Delete removes all restic keys in the bucket. It will not remove the bucket itself.
-func (be *gs) Delete(ctx context.Context) error {
+func (be *Backend) Delete(ctx context.Context) error {
 	return util.DefaultDelete(ctx, be)
 }
 
 // Close does nothing.
-func (be *gs) Close() error { return nil }
+func (be *Backend) Close() error { return nil }
 
 // Warmup not implemented
-func (be *gs) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
+func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
 	return []backend.Handle{}, nil
 }
-func (be *gs) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
+func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }

internal/backend/rclone/backend.go

@@ -28,8 +28,8 @@ import (
 	"golang.org/x/net/http2"
 )
 
-// rclone is used to access data stored somewhere via rclone.
+// Backend is used to access data stored somewhere via rclone.
-type rclone struct {
+type Backend struct {
 	*rest.Backend
 	tr         *http2.Transport
 	cmd        *exec.Cmd
@@ -141,7 +141,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn {
 }
 
 // New initializes a Backend and starts the process.
-func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*rclone, error) {
+func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
 	var (
 		args []string
 		err  error
@@ -196,7 +196,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
 	}
 
 	cmd := stdioConn.cmd
-	be := &rclone{
+	be := &Backend{
 		tr:     tr,
 		cmd:    cmd,
 		waitCh: waitCh,
@@ -270,7 +270,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
 }
 
 // Open starts an rclone process with the given config.
-func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
+func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
 	be, err := newBackend(ctx, cfg, lim, errorLog)
 	if err != nil {
 		return nil, err
@@ -297,7 +297,7 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(st
 }
 
 // Create initializes a new restic repo with rclone.
-func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
+func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
 	be, err := newBackend(ctx, cfg, lim, errorLog)
 	if err != nil {
 		return nil, err
@@ -328,7 +328,7 @@ func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(
 const waitForExit = 5 * time.Second
 
 // Close terminates the backend.
-func (be *rclone) Close() error {
+func (be *Backend) Close() error {
 	debug.Log("exiting rclone")
 	be.tr.CloseIdleConnections()
 
@@ -348,7 +348,7 @@ func (be *rclone) Close() error {
 	return be.waitResult
 }
 
-func (be *rclone) Properties() backend.Properties {
+func (be *Backend) Properties() backend.Properties {
 	properties := be.Backend.Properties()
 	properties.HasFlakyErrors = true
 	return properties

internal/backend/rclone/internal_test.go

@@ -27,7 +27,7 @@ func TestRcloneExit(t *testing.T) {
 		_ = be.Close()
 	}()
 
-	err = be.(*rclone).cmd.Process.Kill()
+	err = be.cmd.Process.Kill()
 	rtest.OK(t, err)
 	t.Log("killed rclone")
 

internal/backend/rest/rest.go

@@ -245,6 +245,7 @@ func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int,
 	}
 
 	if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 && resp.ContentLength != int64(length) {
+		_ = drainAndClose(resp)
 		return nil, &restError{h, http.StatusRequestedRangeNotSatisfiable, "partial out of bounds read"}
 	}
 

internal/backend/s3/s3.go

@@ -25,15 +25,15 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 )
 
-// s3 stores data on an S3 endpoint.
+// Backend stores data on an S3 endpoint.
-type s3 struct {
+type Backend struct {
 	client *minio.Client
 	cfg    Config
 	layout.Layout
 }
 
 // make sure that *Backend implements backend.Backend
-var _ backend.Backend = &s3{}
+var _ backend.Backend = &Backend{}
 
 var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"}
 
@@ -50,7 +50,7 @@ func NewFactory() location.Factory {
 	return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
 }
 
-func open(cfg Config, rt http.RoundTripper) (*s3, error) {
+func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
 	if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) {
@@ -89,7 +89,7 @@ func open(cfg Config, rt http.RoundTripper) (*s3, error) {
 		return nil, errors.Wrap(err, "minio.New")
 	}
 
-	be := &s3{
+	be := &Backend{
 		client: client,
 		cfg:    cfg,
 		Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
@@ -240,12 +240,12 @@ func isAccessDenied(err error) bool {
 }
 
 // IsNotExist returns true if the error is caused by a not existing file.
-func (be *s3) IsNotExist(err error) bool {
+func (be *Backend) IsNotExist(err error) bool {
 	var e minio.ErrorResponse
 	return errors.As(err, &e) && e.Code == "NoSuchKey"
 }
 
-func (be *s3) IsPermanentError(err error) bool {
+func (be *Backend) IsPermanentError(err error) bool {
 	if be.IsNotExist(err) {
 		return true
 	}
@@ -260,7 +260,7 @@ func (be *s3) IsPermanentError(err error) bool {
 	return false
 }
 
-func (be *s3) Properties() backend.Properties {
+func (be *Backend) Properties() backend.Properties {
 	return backend.Properties{
 		Connections:      be.cfg.Connections,
 		HasAtomicReplace: true,
@@ -268,21 +268,26 @@ func (be *s3) Properties() backend.Properties {
 }
 
 // Hasher may return a hash function for calculating a content hash for the backend
-func (be *s3) Hasher() hash.Hash {
+func (be *Backend) Hasher() hash.Hash {
 	return nil
 }
 
+// Path returns the path in the bucket that is used for this backend.
+func (be *Backend) Path() string {
+	return be.cfg.Prefix
+}
+
 // useStorageClass returns whether file should be saved in the provided Storage Class
 // For archive storage classes, only data files are stored using that class; metadata
 // must remain instantly accessible.
-func (be *s3) useStorageClass(h backend.Handle) bool {
+func (be *Backend) useStorageClass(h backend.Handle) bool {
 	isDataFile := h.Type == backend.PackFile && !h.IsMetadata
 	isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass)
 	return !isArchiveClass || isDataFile
 }
 
 // Save stores data in the backend at the handle.
-func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
 	objName := be.Filename(h)
 
 	opts := minio.PutObjectOptions{
@@ -308,14 +313,14 @@ func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReade
 
 // Load runs fn with a reader that yields the contents of the file at h at the
 // given offset.
-func (be *s3) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
 }
 
-func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
+func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
 	objName := be.Filename(h)
 	opts := minio.GetObjectOptions{}
 
@@ -347,7 +352,7 @@ func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offs
 }
 
 // Stat returns information about a blob.
-func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
+func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
 	objName := be.Filename(h)
 	var obj *minio.Object
 
@@ -375,7 +380,7 @@ func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo,
 }
 
 // Remove removes the blob with the given name and type.
-func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
+func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
 	objName := be.Filename(h)
 
 	err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@@ -389,7 +394,7 @@ func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
 
 // List runs fn for each file in the backend which has the type t. When an
 // error occurs (or fn returns an error), List stops and returns it.
-func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
+func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
 	prefix, recursive := be.Basedir(t)
 
 	// make sure prefix ends with a slash
@@ -444,15 +449,15 @@ func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.File
 }
 
 // Delete removes all restic keys in the bucket. It will not remove the bucket itself.
-func (be *s3) Delete(ctx context.Context) error {
+func (be *Backend) Delete(ctx context.Context) error {
 	return util.DefaultDelete(ctx, be)
 }
 
 // Close does nothing
-func (be *s3) Close() error { return nil }
+func (be *Backend) Close() error { return nil }
 
 // Warmup transitions handles from cold to hot storage if needed.
-func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
+func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
 	handlesWarmingUp := []backend.Handle{}
 
 	if be.cfg.EnableRestore {
@@ -473,7 +478,7 @@ func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.H
 }
 
 // requestRestore sends a glacier restore request on a given file.
-func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error) {
+func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) {
 	objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{})
 	if err != nil {
 		return false, err
@@ -509,7 +514,7 @@ func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error)
 }
 
 // getWarmupStatus returns the warmup status of the provided object.
-func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
+func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
 	// We can't use objectInfo.StorageClass to get the storage class of the
 	// object because this field is only set during ListObjects operations.
 	// The response header is the documented way to get the storage class
@@ -540,7 +545,7 @@ func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
 }
 
 // WarmupWait waits until all handles are in hot storage.
-func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
+func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error {
 	timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout)
 	defer timeoutCtxCancel()
 
@@ -559,7 +564,7 @@ func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
 }
 
 // waitForRestore waits for a given file to be restored.
-func (be *s3) waitForRestore(ctx context.Context, filename string) error {
+func (be *Backend) waitForRestore(ctx context.Context, filename string) error {
 	for {
 		var objectInfo minio.ObjectInfo
 

internal/backend/sftp/sftp.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -37,7 +38,8 @@ type SFTP struct {
 	cmd    *exec.Cmd
 	result <-chan error
 
-	posixRename bool
+	posixRename       bool
+	chmodBeforeRemove atomic.Bool
 
 	layout.Layout
 	Config
@@ -363,6 +365,7 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
 	if err == nil {
 		err = f.Chmod(r.Modes.File)
 		if err != nil {
+			_ = f.Close()
 			return errors.Wrapf(err, "Chmod %v", tmpFilename)
 		}
 	}
@@ -404,12 +407,11 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader
 	} else {
 		err = r.c.Rename(tmpFilename, filename)
 	}
-	err = setFileReadonly(r.c, filename, r.Modes.File)
 	if err != nil {
-		return errors.Errorf("sftp setFileReadonly: %v", err)
+		return errors.Wrapf(err, "Rename %v", tmpFilename)
 	}
-
+	err = setFileReadonly(r.c, filename, r.Modes.File)
-	return errors.Wrapf(err, "Rename %v", tmpFilename)
+	return errors.Wrapf(err, "setFileReadonly %v", filename)
 }
 
 // checkNoSpace checks if err was likely caused by lack of available space
@@ -507,7 +509,37 @@ func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
 		return err
 	}
 
-	return errors.Wrapf(r.c.Remove(r.Filename(h)), "Remove %v", r.Filename(h))
+	path := r.Filename(h)
+
+	if r.chmodBeforeRemove.Load() {
+		return r.removeWithChmod(path)
+	}
+
+	// optimistically try to remove the file
+	err := r.c.Remove(path)
+	if err == nil {
+		return nil
+	}
+	if !errors.Is(err, os.ErrPermission) {
+		return errors.Wrapf(err, "Remove %v", path)
+	}
+
+	// fallback to chmod + remove
+	// this is necessary on Windows where read-only files cannot be deleted without chmod.
+	if err := r.removeWithChmod(path); err != nil {
+		return err
+	}
+	r.chmodBeforeRemove.Store(true)
+	return nil
+}
+
+func (r *SFTP) removeWithChmod(path string) error {
+	err := r.c.Chmod(path, r.Modes.File)
+	if err != nil {
+		return errors.Wrapf(err, "Chmod %v", path)
+	}
+
+	return errors.Wrapf(r.c.Remove(path), "Remove %v", path)
 }
 
 // List runs fn for each file in the backend which has the type t. When an

internal/backend/test/tests.go

@@ -251,6 +251,10 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
 	test.OK(t, b.Remove(context.TODO(), handle))
 }
 
+type setter interface {
+	SetListMaxItems(int)
+}
+
 // TestList makes sure that the backend implements List() pagination correctly.
 func (s *Suite[C]) TestList(t *testing.T) {
 	random := seedRand(t)
@@ -298,39 +302,54 @@ func (s *Suite[C]) TestList(t *testing.T) {
 
 	t.Logf("wrote %v files", len(list1))
 
-	list2 := make(map[restic.ID]int64)
+	var tests = []struct {
-
+		maxItems int
-	err = b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
+	}{
-		id, err := restic.ParseID(fi.Name)
+		{11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123},
-		if err != nil {
-			t.Fatal(err)
-		}
-		list2[id] = fi.Size
-		return nil
-	})
-
-	if err != nil {
-		t.Fatalf("List returned error %v", err)
 	}
 
-	t.Logf("loaded %v IDs from backend", len(list2))
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
+			list2 := make(map[restic.ID]int64)
 
-	for id, size := range list1 {
+			if s, ok := b.(setter); ok {
-		size2, ok := list2[id]
+				t.Logf("setting max list items to %d", test.maxItems)
-		if !ok {
+				s.SetListMaxItems(test.maxItems)
-			t.Errorf("id %v not returned by List()", id.Str())
+			}
-		}
 
-		if size != size2 {
+			err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
-			t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
+				id, err := restic.ParseID(fi.Name)
-		}
+				if err != nil {
-	}
+					t.Fatal(err)
+				}
+				list2[id] = fi.Size
+				return nil
+			})
 
-	for id := range list2 {
+			if err != nil {
-		_, ok := list1[id]
+				t.Fatalf("List returned error %v", err)
-		if !ok {
+			}
-			t.Errorf("extra id %v returned by List()", id.Str())
+
-		}
+			t.Logf("loaded %v IDs from backend", len(list2))
+
+			for id, size := range list1 {
+				size2, ok := list2[id]
+				if !ok {
+					t.Errorf("id %v not returned by List()", id.Str())
+				}
+
+				if size != size2 {
+					t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
+				}
+			}
+
+			for id := range list2 {
+				_, ok := list1[id]
+				if !ok {
+					t.Errorf("extra id %v returned by List()", id.Str())
+				}
+			}
+		})
 	}
 
 	t.Logf("remove %d files", numTestFiles)

internal/backend/util/defaults.go

@@ -35,10 +35,15 @@ func DefaultDelete(ctx context.Context, be backend.Backend) error {
 
 	for _, t := range alltypes {
 		err := be.List(ctx, t, func(fi backend.FileInfo) error {
-			return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
+			err := be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
+			if err != nil && be.IsNotExist(err) {
+				// deletion of files created by TestSaveError may happen with a delay for the REST server, so we ignore the error
+				err = nil
+			}
+			return err
 		})
 		if err != nil {
-			return nil
+			return err
 		}
 	}
 	err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})

internal/checker/checker_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -18,6 +19,7 @@ import (
 	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/repository/hashing"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
 )
@@ -73,8 +75,7 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) {
 }
 
 func TestCheckRepo(t *testing.T) {
-	repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
+	repo, _ := repository.TestFromFixture(t, checkerTestData)
-	defer cleanup()
 
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -91,8 +92,7 @@ func TestCheckRepo(t *testing.T) {
 }
 
 func TestMissingPack(t *testing.T) {
-	repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
+	repo, be := repository.TestFromFixture(t, checkerTestData)
-	defer cleanup()
 
 	packID := restic.TestParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6")
 	test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: packID.String()}))
@@ -109,7 +109,7 @@ func TestMissingPack(t *testing.T) {
 	test.Assert(t, len(errs) == 1,
 		"expected exactly one error, got %v", len(errs))
 
-	if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
+	if err, ok := errs[0].(*repository.PackError); ok {
 		test.Equals(t, packID, err.ID)
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -117,8 +117,7 @@ func TestMissingPack(t *testing.T) {
 }
 
 func TestUnreferencedPack(t *testing.T) {
-	repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
+	repo, be := repository.TestFromFixture(t, checkerTestData)
-	defer cleanup()
 
 	// index 3f1a only references pack 60e0
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
@@ -137,7 +136,7 @@ func TestUnreferencedPack(t *testing.T) {
 	test.Assert(t, len(errs) == 1,
 		"expected exactly one error, got %v", len(errs))
 
-	if err, ok := errs[0].(*repository.ErrPackMetadata); ok {
+	if err, ok := errs[0].(*repository.PackError); ok {
 		test.Equals(t, packID, err.ID.String())
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
@@ -145,8 +144,7 @@ func TestUnreferencedPack(t *testing.T) {
 }
 
 func TestUnreferencedBlobs(t *testing.T) {
-	repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
+	repo, be := repository.TestFromFixture(t, checkerTestData)
-	defer cleanup()
 
 	snapshotID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
 	test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.SnapshotFile, Name: snapshotID.String()}))
@@ -180,8 +178,7 @@ func TestUnreferencedBlobs(t *testing.T) {
 }
 
 func TestModifiedIndex(t *testing.T) {
-	repo, be, cleanup := repository.TestFromFixture(t, checkerTestData)
+	repo, be := repository.TestFromFixture(t, checkerTestData)
-	defer cleanup()
 
 	done := make(chan struct{})
 	defer close(done)
@@ -190,19 +187,56 @@ func TestModifiedIndex(t *testing.T) {
 		Type: restic.IndexFile,
 		Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
 	}
-	var data []byte
+
-	test.OK(t, be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
+	tmpfile, err := os.CreateTemp("", "restic-test-mod-index-")
-		var err error
+	if err != nil {
-		data, err = io.ReadAll(rd)
+		t.Fatal(err)
+	}
+	defer func() {
+		err := tmpfile.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = os.Remove(tmpfile.Name())
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	wr := io.Writer(tmpfile)
+	var hw *hashing.Writer
+	if be.Hasher() != nil {
+		hw = hashing.NewWriter(wr, be.Hasher())
+		wr = hw
+	}
+
+	// read the file from the backend
+	err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error {
+		_, err := io.Copy(wr, rd)
 		return err
-	}))
+	})
+	test.OK(t, err)
+
 	// save the index again with a modified name so that the hash doesn't match
 	// the content any more
 	h2 := backend.Handle{
 		Type: restic.IndexFile,
 		Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
 	}
-	test.OK(t, be.Save(context.TODO(), h2, backend.NewByteReader(data, be.Hasher())))
+
+	var hash []byte
+	if hw != nil {
+		hash = hw.Sum(nil)
+	}
+	rd, err := backend.NewFileReader(tmpfile, hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = be.Save(context.TODO(), h2, rd)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -220,8 +254,7 @@ func TestModifiedIndex(t *testing.T) {
 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 
 func TestDuplicatePacksInIndex(t *testing.T) {
-	repo, _, cleanup := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
+	repo, _ := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
-	defer cleanup()
 
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -420,8 +453,7 @@ func (r *loadTreesOnceRepository) LoadBlob(ctx context.Context, t restic.BlobTyp
 }
 
 func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
-	repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
+	repo, _ := repository.TestFromFixture(t, checkerTestData)
-	defer cleanup()
 	checkRepo := &loadTreesOnceRepository{
 		Repository:  repo,
 		loadedTrees: restic.NewIDSet(),
@@ -569,13 +601,12 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
 	test.Assert(t, delayRepo.Triggered, "delay repository did not trigger")
 }
 
-func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, func()) {
+func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository) {
-	repo, _, cleanup := repository.TestFromFixture(t, checkerTestData)
+	repo, _ := repository.TestFromFixture(t, checkerTestData)
 
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
 	if len(errs) > 0 {
-		defer cleanup()
 		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
 	}
 
@@ -584,12 +615,11 @@ func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, fun
 			t.Fatalf("expected mixed pack hint, got %v", err)
 		}
 	}
-	return chkr, repo, cleanup
+	return chkr, repo
 }
 
 func BenchmarkChecker(t *testing.B) {
-	chkr, _, cleanup := loadBenchRepository(t)
+	chkr, _ := loadBenchRepository(t)
-	defer cleanup()
 
 	t.ResetTimer()
 
@@ -601,8 +631,7 @@ func BenchmarkChecker(t *testing.B) {
 }
 
 func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) {
-	chkr, repo, cleanup := loadBenchRepository(t)
+	chkr, repo := loadBenchRepository(t)
-	defer cleanup()
 
 	snID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
 	sn2, err := data.LoadSnapshot(context.TODO(), repo, snID)

internal/fs/node_xattr.go

@@ -109,7 +109,7 @@ func nodeFillExtendedAttributes(node *data.Node, path string, ignoreListError bo
 	for _, attr := range xattrs {
 		attrVal, err := getxattr(path, attr)
 		if err != nil {
-			warnf("can not obtain extended attribute %v for %v:\n", attr, path)
+			warnf("can not obtain extended attribute %v for %v: %v\n", attr, path, err)
 			continue
 		}
 		attr := data.ExtendedAttribute{

internal/fuse/dir.go

@@ -225,7 +225,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		return nil, err
 	}
 
-	return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
+	return d.cache.lookupOrCreate(name, -1, func(forget forgetFn) (fs.Node, error) {
 		node, ok := d.items[name]
 		if !ok {
 			debug.Log("  Lookup(%v) -> not found", name)

internal/fuse/fuse_test.go

@@ -247,6 +247,39 @@ func TestStableNodeObjects(t *testing.T) {
 	testStableLookup(t, dir, "file-2")
 }
 
+func TestSnapshotsDirLatestSymlinkUpdatesAfterReload(t *testing.T) {
+	repo := repository.TestRepository(t)
+	timeTemplate := "2006-01-02T15:04:05"
+
+	firstTime, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56Z")
+	rtest.OK(t, err)
+	secondTime := firstTime.Add(time.Hour)
+
+	data.TestCreateSnapshot(t, repo, firstTime, 0)
+	root := NewRoot(repo, Config{
+		TimeTemplate:  timeTemplate,
+		PathTemplates: []string{"snapshots/%T"},
+	})
+
+	snapshotsDir := testStableLookup(t, root, "snapshots")
+	rtest.Equals(t, firstTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
+
+	data.TestCreateSnapshot(t, repo, secondTime, 0)
+	root.SnapshotsDir.dirStruct.lastCheck = time.Now().Add(-2 * minSnapshotsReloadTime)
+
+	rtest.Equals(t, secondTime.Format(timeTemplate), readLatestTarget(t, snapshotsDir))
+}
+
+func readLatestTarget(t testing.TB, node fs.Node) string {
+	t.Helper()
+
+	latest, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), "latest")
+	rtest.OK(t, err)
+	target, err := latest.(fs.NodeReadlinker).Readlink(context.TODO(), nil)
+	rtest.OK(t, err)
+	return target
+}
+
 // Test reporting of fuse.Attr.Blocks in multiples of 512.
 func TestBlocks(t *testing.T) {
 	root := &Root{}

internal/fuse/snapshots_dir.go

@@ -61,7 +61,7 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	debug.Log("ReadDirAll()")
 
 	// update snapshots
-	meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
+	meta, _, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
 	if err != nil {
 		return nil, unwrapCtxCanceled(err)
 	} else if meta == nil {
@@ -104,14 +104,14 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 	debug.Log("Lookup(%s)", name)
 
-	meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
+	meta, gen, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
 	if err != nil {
 		return nil, unwrapCtxCanceled(err)
 	} else if meta == nil {
 		return nil, syscall.ENOENT
 	}
 
-	return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
+	return d.cache.lookupOrCreate(name, gen, func(forget forgetFn) (fs.Node, error) {
 		entry := meta.names[name]
 		if entry == nil {
 			return nil, syscall.ENOENT

internal/fuse/snapshots_dirstruct.go

@@ -42,6 +42,10 @@ type SnapshotsDirStructure struct {
 
 	hash      [sha256.Size]byte // Hash at last check.
 	lastCheck time.Time
+
+	// generation is incremented whenever the directory structure is rebuilt.
+	// It allows treeCache instances to detect stale entries and reset themselves.
+	generation int64
 }
 
 // NewSnapshotsDirStructure returns a new directory structure for snapshots.
@@ -280,6 +284,7 @@ func (d *SnapshotsDirStructure) makeDirs(snapshots data.Snapshots) {
 	}
 
 	d.entries = entries
+	d.generation++
 }
 
 const minSnapshotsReloadTime = 60 * time.Second
@@ -337,13 +342,13 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
 	return nil
 }
 
-func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) {
+func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, int64, error) {
 	err := d.updateSnapshots(ctx)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	d.mutex.Lock()
 	defer d.mutex.Unlock()
-	return d.entries[prefix], nil
+	return d.entries[prefix], d.generation, nil
 }

internal/fuse/tree_cache.go

@@ -5,12 +5,15 @@ package fuse
 import (
 	"sync"
 
+	"github.com/restic/restic/internal/debug"
+
 	"github.com/anacrolix/fuse/fs"
 )
 
 type treeCache struct {
-	nodes map[string]fs.Node
+	nodes      map[string]fs.Node
-	m     sync.Mutex
+	m          sync.Mutex
+	generation int64
 }
 
 type forgetFn func()
@@ -21,19 +24,28 @@ func newTreeCache() *treeCache {
 	}
 }
 
-func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
+func (t *treeCache) lookupOrCreate(name string, generation int64, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
 	t.m.Lock()
 	defer t.m.Unlock()
 
+	if generation >= 0 && generation != t.generation {
+		debug.Log("treeCache generation changed %d -> %d, resetting cache", t.generation, generation)
+		t.nodes = make(map[string]fs.Node)
+		t.generation = generation
+	}
+
 	if node, ok := t.nodes[name]; ok {
 		return node, nil
 	}
 
+	cacheGeneration := t.generation
 	node, err := create(func() {
 		t.m.Lock()
 		defer t.m.Unlock()
 
-		delete(t.nodes, name)
+		if t.generation == cacheGeneration {
+			delete(t.nodes, name)
+		}
 	})
 	if err != nil {
 		return nil, err

internal/fuse/tree_cache_test.go

@@ -0,0 +1,82 @@
+//go:build darwin || freebsd || linux
+
+package fuse
+
+import (
+	"context"
+	"testing"
+
+	"github.com/anacrolix/fuse"
+	"github.com/anacrolix/fuse/fs"
+
+	"github.com/restic/restic/internal/test"
+)
+
+type cacheTestNode struct {
+	id int
+}
+
+func (n cacheTestNode) Attr(context.Context, *fuse.Attr) error {
+	return nil
+}
+
+func TestTreeCacheGeneration(t *testing.T) {
+	cache := newTreeCache()
+	created := 0
+	create := func(forgetFn) (fs.Node, error) {
+		created++
+		return cacheTestNode{id: created}, nil
+	}
+
+	node1, err := cache.lookupOrCreate("node", 1, create)
+	test.OK(t, err)
+	node2, err := cache.lookupOrCreate("node", 1, create)
+	test.OK(t, err)
+	test.Assert(t, node1 == node2, "lookup should reuse cached node")
+	test.Equals(t, 1, created)
+
+	node3, err := cache.lookupOrCreate("node", 2, create)
+	test.OK(t, err)
+	test.Assert(t, node1 != node3, "lookup should recreate node after generation change")
+	test.Equals(t, 2, created)
+
+	node4, err := cache.lookupOrCreate("node", -1, create)
+	test.OK(t, err)
+	test.Assert(t, node3 == node4, "negative generation should not reset cache")
+	test.Equals(t, 2, created)
+
+	node5, err := cache.lookupOrCreate("node", 3, create)
+	test.OK(t, err)
+	test.Assert(t, node4 != node5, "lookup should still track later generation changes")
+	test.Equals(t, 3, created)
+}
+
+func TestTreeCacheForgetOnlyRemovesSameGeneration(t *testing.T) {
+	cache := newTreeCache()
+	created := 0
+	var forgets []forgetFn
+	create := func(forget forgetFn) (fs.Node, error) {
+		created++
+		forgets = append(forgets, forget)
+		return cacheTestNode{id: created}, nil
+	}
+
+	node1, err := cache.lookupOrCreate("node", 1, create)
+	test.OK(t, err)
+	node2, err := cache.lookupOrCreate("node", 2, create)
+	test.OK(t, err)
+	test.Assert(t, node1 != node2, "lookup should recreate node after generation change")
+	test.Equals(t, 2, created)
+
+	forgets[0]()
+	node3, err := cache.lookupOrCreate("node", 2, create)
+	test.OK(t, err)
+	test.Assert(t, node2 == node3, "forget from an old generation must not remove the current node")
+	test.Equals(t, 2, created)
+
+	forgets[1]()
+	node4, err := cache.lookupOrCreate("node", 2, create)
+	test.OK(t, err)
+	test.Assert(t, node3 != node4, "forget from the current generation should remove the cached node")
+	test.Equals(t, 3, created)
+}

internal/global/global.go

@@ -35,7 +35,7 @@ import (
 // to a missing backend storage location or config file
 var ErrNoRepository = errors.New("repository does not exist")
 
-const Version = "0.19.0-dev (compiled manually)"
+const Version = "0.19.1-dev (compiled manually)"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"

internal/repository/check.go

@@ -0,0 +1,215 @@
+package repository
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository/hashing"
+	"github.com/restic/restic/internal/repository/pack"
+	"github.com/restic/restic/internal/restic"
+)
+
+// ErrPackData is returned if errors are discovered while verifying a packfile
+type ErrPackData struct {
+	PackID restic.ID
+	errs   []error
+}
+
+func (e *ErrPackData) Error() string {
+	return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
+}
+
+type partialReadError struct {
+	err error
+}
+
+func (e *partialReadError) Error() string {
+	return e.err.Error()
+}
+
+// CheckPack reads a pack and checks the integrity of all blobs.
+func CheckPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
+	err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
+	if err != nil {
+		if r.cache != nil {
+			// ignore error as there's not much we can do here
+			_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
+		}
+
+		// retry pack verification to detect transient errors
+		err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
+		if err2 != nil {
+			err = err2
+		} else {
+			err = fmt.Errorf("check successful on second attempt, original error %w", err)
+		}
+	}
+	return err
+}
+
+func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
+
+	debug.Log("checking pack %v", id.String())
+
+	if len(blobs) == 0 {
+		return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
+	}
+
+	// sanity check blobs in index
+	blobs.Sort()
+	idxHdrSize := pack.CalculateHeaderSize(blobs)
+	lastBlobEnd := 0
+	nonContinuousPack := false
+	for _, blob := range blobs {
+		if lastBlobEnd != int(blob.Offset) {
+			nonContinuousPack = true
+		}
+		lastBlobEnd = int(blob.Offset + blob.Length)
+	}
+	// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
+
+	var errs []error
+	if nonContinuousPack {
+		debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
+		errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
+	}
+
+	// calculate hash on-the-fly while reading the pack and capture pack header
+	var hash restic.ID
+	var hdrBuf []byte
+	// must use a separate slice from `errs` here as we're only interested in the last retry
+	var blobErrors []error
+	h := backend.Handle{Type: backend.PackFile, Name: id.String()}
+	err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
+		hrd := hashing.NewReader(rd, sha256.New())
+		bufRd.Reset(hrd)
+		// reset blob errors for each retry
+		blobErrors = nil
+
+		it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
+		for {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+
+			val, err := it.Next()
+			if err == errPackEOF {
+				break
+			} else if err != nil {
+				return &partialReadError{err}
+			}
+			debug.Log("  check blob %v: %v", val.Handle.ID, val.Handle)
+			if val.Err != nil {
+				debug.Log("  error verifying blob %v: %v", val.Handle.ID, val.Err)
+				blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
+			}
+		}
+
+		// skip enough bytes until we reach the possible header start
+		curPos := lastBlobEnd
+		minHdrStart := int(size) - pack.MaxHeaderSize
+		if minHdrStart > curPos {
+			_, err := bufRd.Discard(minHdrStart - curPos)
+			if err != nil {
+				return &partialReadError{err}
+			}
+			curPos += minHdrStart - curPos
+		}
+
+		// read remainder, which should be the pack header
+		var err error
+		hdrBuf = make([]byte, int(size-int64(curPos)))
+		_, err = io.ReadFull(bufRd, hdrBuf)
+		if err != nil {
+			return &partialReadError{err}
+		}
+
+		hash = restic.IDFromHash(hrd.Sum(nil))
+		return nil
+	})
+	errs = append(errs, blobErrors...)
+	if err != nil {
+		var e *partialReadError
+		isPartialReadError := errors.As(err, &e)
+		// failed to load the pack file, return as further checks cannot succeed anyways
+		debug.Log("  error streaming pack (partial %v): %v", isPartialReadError, err)
+		if isPartialReadError {
+			return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
+		}
+
+		// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
+		// completely failed to download such that there's no point in repairing anything.
+		return fmt.Errorf("download error: %w", err)
+	}
+	if !hash.Equal(id) {
+		debug.Log("pack ID does not match, want %v, got %v", id, hash)
+		return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
+	}
+
+	blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
+	if err != nil {
+		return &ErrPackData{PackID: id, errs: append(errs, err)}
+	}
+
+	if uint32(idxHdrSize) != hdrSize {
+		debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
+		errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
+	}
+
+	for _, blob := range blobs {
+		// Check if blob is contained in index and position is correct
+		idxHas := false
+		for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
+			if pb.PackID == id && pb.Blob == blob {
+				idxHas = true
+				break
+			}
+		}
+		if !idxHas {
+			errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
+			continue
+		}
+	}
+
+	if len(errs) > 0 {
+		return &ErrPackData{PackID: id, errs: errs}
+	}
+
+	return nil
+}
+
+type bufReader struct {
+	rd  *bufio.Reader
+	buf []byte
+}
+
+func newBufReader(rd *bufio.Reader) *bufReader {
+	return &bufReader{
+		rd: rd,
+	}
+}
+
+func (b *bufReader) Discard(n int) (discarded int, err error) {
+	return b.rd.Discard(n)
+}
+
+func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
+	if cap(b.buf) < n {
+		b.buf = make([]byte, n)
+	}
+	b.buf = b.buf[:n]
+
+	_, err = io.ReadFull(b.rd, b.buf)
+	if err != nil {
+		return nil, err
+	}
+	return b.buf, nil
+}

internal/repository/checker.go

@@ -2,17 +2,12 @@ package repository
 
 import (
 	"bufio"
-	"bytes"
 	"context"
-	"crypto/sha256"
 	"fmt"
-	"io"
 
 	"github.com/klauspost/compress/zstd"
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository/hashing"
 	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
@@ -51,29 +46,18 @@ func (e *ErrMixedPack) Error() string {
 	return fmt.Sprintf("pack %v contains a mix of tree and data blobs", e.PackID.Str())
 }
 
-// ErrPackMetadata describes an error with a specific pack. It is used for missing, truncated or orphaned packs.
+// PackError describes an error with a specific pack.
-// Errors of the actual pack data are returned as ErrPackData.
+type PackError struct {
-type ErrPackMetadata struct {
 	ID        restic.ID
 	Orphaned  bool
 	Truncated bool
 	Err       error
 }
 
-func (e *ErrPackMetadata) Error() string {
+func (e *PackError) Error() string {
 	return "pack " + e.ID.String() + ": " + e.Err.Error()
 }
 
-// ErrPackData is returned if errors are discovered while verifying a packfile
-type ErrPackData struct {
-	PackID restic.ID
-	errs   []error
-}
-
-func (e *ErrPackData) Error() string {
-	return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs)
-}
-
 // Checker handles index-related operations for repository checking.
 type Checker struct {
 	repo *Repository
@@ -215,7 +199,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
 			select {
 			case <-ctx.Done():
 				return
-			case errChan <- &ErrPackMetadata{ID: id, Err: errors.New("does not exist")}:
+			case errChan <- &PackError{ID: id, Err: errors.New("does not exist")}:
 			}
 			continue
 		}
@@ -225,7 +209,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
 			select {
 			case <-ctx.Done():
 				return
-			case errChan <- &ErrPackMetadata{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
+			case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}:
 			}
 		}
 	}
@@ -235,7 +219,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
 		select {
 		case <-ctx.Done():
 			return
-		case errChan <- &ErrPackMetadata{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
+		case errChan <- &PackError{ID: orphanID, Orphaned: true, Err: errors.New("not referenced in any index")}:
 		}
 	}
 }
@@ -269,7 +253,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
 			bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize)
 			dec, err := zstd.NewReader(nil)
 			if err != nil {
-				panic(dec)
+				panic(err)
 			}
 			defer dec.Close()
 			for {
@@ -285,7 +269,7 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
 					}
 				}
 
-				err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
+				err := CheckPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec)
 				p.Add(1)
 				if err == nil {
 					continue
@@ -325,186 +309,3 @@ func (c *Checker) ReadPacks(ctx context.Context, filter func(packs map[restic.ID
 		}
 	}
 }
-
-// checkPack reads a pack and checks the integrity of all blobs.
-func checkPack(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
-	err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
-	if err != nil {
-		if r.cache != nil {
-			// ignore error as there's not much we can do here
-			_ = r.cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()})
-		}
-
-		// retry pack verification to detect transient errors
-		err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec)
-		if err2 != nil {
-			err = err2
-		} else {
-			err = fmt.Errorf("check successful on second attempt, original error %w", err)
-		}
-	}
-	return err
-}
-
-func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs restic.Blobs, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error {
-
-	type partialReadError struct {
-		error
-	}
-
-	debug.Log("checking pack %v", id.String())
-
-	if len(blobs) == 0 {
-		return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}}
-	}
-
-	// sanity check blobs in index
-	blobs.Sort()
-	idxHdrSize := pack.CalculateHeaderSize(blobs)
-	lastBlobEnd := 0
-	nonContinuousPack := false
-	for _, blob := range blobs {
-		if lastBlobEnd != int(blob.Offset) {
-			nonContinuousPack = true
-		}
-		lastBlobEnd = int(blob.Offset + blob.Length)
-	}
-	// size was calculated by masterindex.PackSize, thus there's no need to recalculate it here
-
-	var errs []error
-	if nonContinuousPack {
-		debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs)
-		errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs"))
-	}
-
-	// calculate hash on-the-fly while reading the pack and capture pack header
-	var hash restic.ID
-	var hdrBuf []byte
-	// must use a separate slice from `errs` here as we're only interested in the last retry
-	var blobErrors []error
-	h := backend.Handle{Type: backend.PackFile, Name: id.String()}
-	err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error {
-		hrd := hashing.NewReader(rd, sha256.New())
-		bufRd.Reset(hrd)
-		// reset blob errors for each retry
-		blobErrors = nil
-
-		it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec)
-		for {
-			if ctx.Err() != nil {
-				return ctx.Err()
-			}
-
-			val, err := it.Next()
-			if err == errPackEOF {
-				break
-			} else if err != nil {
-				return &partialReadError{err}
-			}
-			debug.Log("  check blob %v: %v", val.Handle.ID, val.Handle)
-			if val.Err != nil {
-				debug.Log("  error verifying blob %v: %v", val.Handle.ID, val.Err)
-				blobErrors = append(blobErrors, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err))
-			}
-		}
-
-		// skip enough bytes until we reach the possible header start
-		curPos := lastBlobEnd
-		minHdrStart := int(size) - pack.MaxHeaderSize
-		if minHdrStart > curPos {
-			_, err := bufRd.Discard(minHdrStart - curPos)
-			if err != nil {
-				return &partialReadError{err}
-			}
-			curPos += minHdrStart - curPos
-		}
-
-		// read remainder, which should be the pack header
-		var err error
-		hdrBuf = make([]byte, int(size-int64(curPos)))
-		_, err = io.ReadFull(bufRd, hdrBuf)
-		if err != nil {
-			return &partialReadError{err}
-		}
-
-		hash = restic.IDFromHash(hrd.Sum(nil))
-		return nil
-	})
-	errs = append(errs, blobErrors...)
-	if err != nil {
-		var e *partialReadError
-		isPartialReadError := errors.As(err, &e)
-		// failed to load the pack file, return as further checks cannot succeed anyways
-		debug.Log("  error streaming pack (partial %v): %v", isPartialReadError, err)
-		if isPartialReadError {
-			return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))}
-		}
-
-		// The check command suggests to repair files for which a `ErrPackData` is returned. However, this file
-		// completely failed to download such that there's no point in repairing anything.
-		return fmt.Errorf("download error: %w", err)
-	}
-	if !hash.Equal(id) {
-		debug.Log("pack ID does not match, want %v, got %v", id, hash)
-		return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))}
-	}
-
-	blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf)))
-	if err != nil {
-		return &ErrPackData{PackID: id, errs: append(errs, err)}
-	}
-
-	if uint32(idxHdrSize) != hdrSize {
-		debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)
-		errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize))
-	}
-
-	for _, blob := range blobs {
-		// Check if blob is contained in index and position is correct
-		idxHas := false
-		for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) {
-			if pb.PackID == id && pb.Blob == blob {
-				idxHas = true
-				break
-			}
-		}
-		if !idxHas {
-			errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID))
-			continue
-		}
-	}
-
-	if len(errs) > 0 {
-		return &ErrPackData{PackID: id, errs: errs}
-	}
-
-	return nil
-}
-
-type bufReader struct {
-	rd  *bufio.Reader
-	buf []byte
-}
-
-func newBufReader(rd *bufio.Reader) *bufReader {
-	return &bufReader{
-		rd: rd,
-	}
-}
-
-func (b *bufReader) Discard(n int) (discarded int, err error) {
-	return b.rd.Discard(n)
-}
-
-func (b *bufReader) ReadFull(n int) (buf []byte, err error) {
-	if cap(b.buf) < n {
-		b.buf = make([]byte, n)
-	}
-	b.buf = b.buf[:n]
-
-	_, err = io.ReadFull(b.rd, b.buf)
-	if err != nil {
-		return nil, err
-	}
-	return b.buf, nil
-}

internal/repository/debug.go

@@ -1,390 +0,0 @@
-//go:build debug
-
-package repository
-
-import (
-	"context"
-	"crypto/aes"
-	"crypto/cipher"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-	"runtime"
-	"sync"
-	"time"
-
-	"github.com/klauspost/compress/zstd"
-	"golang.org/x/sync/errgroup"
-
-	"github.com/restic/restic/internal/crypto"
-	"github.com/restic/restic/internal/repository/index"
-	"github.com/restic/restic/internal/repository/pack"
-	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/progress"
-)
-
-type packDumpEntry struct {
-	Name  string         `json:"name"`
-	Blobs []packDumpBlob `json:"blobs"`
-}
-
-type packDumpBlob struct {
-	Type   restic.BlobType `json:"type"`
-	Length uint            `json:"length"`
-	ID     restic.ID       `json:"id"`
-	Offset uint            `json:"offset"`
-}
-
-func writePackDumpJSON(wr io.Writer, item any) error {
-	buf, err := json.MarshalIndent(item, "", "  ")
-	if err != nil {
-		return err
-	}
-	_, err = wr.Write(append(buf, '\n'))
-	return err
-}
-
-// DumpPacks lists each pack file and writes its header blob layout as JSON to wr.
-func DumpPacks(ctx context.Context, repo *Repository, wr io.Writer, printer progress.Printer) error {
-	var m sync.Mutex
-	return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
-		blobs, err := repo.ListPack(ctx, id, size)
-		if err != nil {
-			printer.E("error for pack %v: %v", id.Str(), err)
-			return nil
-		}
-
-		p := packDumpEntry{
-			Name:  id.String(),
-			Blobs: make([]packDumpBlob, len(blobs)),
-		}
-		for i, blob := range blobs {
-			p.Blobs[i] = packDumpBlob{
-				Type:   blob.Type,
-				Length: blob.Length,
-				ID:     blob.ID,
-				Offset: blob.Offset,
-			}
-		}
-
-		m.Lock()
-		defer m.Unlock()
-		return writePackDumpJSON(wr, p)
-	})
-}
-
-// DumpIndexes loads each on-disk index file and writes its debug dump to wr.
-func DumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
-	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
-		printer.S("index_id: %v", id)
-		if err != nil {
-			return err
-		}
-
-		return idx.Dump(wr)
-	})
-}
-
-// ExaminePackOptions configures debug examination of a pack file.
-type ExaminePackOptions struct {
-	TryRepair     bool
-	RepairByte    bool
-	ExtractPack   bool
-	ReuploadBlobs bool
-}
-
-// ExaminePack loads and inspects a pack file and its index entries.
-func ExaminePack(ctx context.Context, repo *Repository, id restic.ID, opts ExaminePackOptions, printer progress.Printer) error {
-	printer.S("examine %v", id)
-
-	buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
-	// also process damaged pack files
-	if buf == nil {
-		return err
-	}
-	printer.S("  file size is %v", len(buf))
-	gotID := restic.Hash(buf)
-	if !id.Equal(gotID) {
-		printer.S("  wanted hash %v, got %v", id, gotID)
-	} else {
-		printer.S("  hash for file content matches")
-	}
-
-	printer.S("  ========================================")
-	printer.S("  looking for info in the indexes")
-
-	blobsLoaded := false
-	// examine all data the indexes have for the pack file
-	for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
-		blobs := b.Blobs
-		if len(blobs) == 0 {
-			continue
-		}
-
-		checkPackSize(blobs, len(buf), printer)
-
-		err = loadBlobs(ctx, opts, repo, id, blobs, printer)
-		if err != nil {
-			printer.E("error: %v", err)
-		} else {
-			blobsLoaded = true
-		}
-	}
-
-	printer.S("  ========================================")
-	printer.S("  inspect the pack itself")
-
-	blobs, err := repo.ListPack(ctx, id, int64(len(buf)))
-	if err != nil {
-		return fmt.Errorf("pack %v: %v", id.Str(), err)
-	}
-	checkPackSize(blobs, len(buf), printer)
-
-	if !blobsLoaded {
-		return loadBlobs(ctx, opts, repo, id, blobs, printer)
-	}
-	return nil
-}
-
-func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
-	// track current size and offset
-	var size, offset uint64
-
-	blobs.Sort()
-
-	for _, pb := range blobs {
-		printer.S("      %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
-		if offset != uint64(pb.Offset) {
-			printer.S("      hole in file, want offset %v, got %v", offset, pb.Offset)
-		}
-		offset = uint64(pb.Offset + pb.Length)
-		size += uint64(pb.Length)
-	}
-	size += uint64(pack.CalculateHeaderSize(blobs))
-
-	if uint64(fileSize) != size {
-		printer.S("      file sizes do not match: computed %v, file size is %v", size, fileSize)
-	} else {
-		printer.S("      file sizes match")
-	}
-}
-
-func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
-	if bytewise {
-		printer.S("        trying to repair blob by finding a broken byte")
-	} else {
-		printer.S("        trying to repair blob with single bit flip")
-	}
-
-	ch := make(chan int)
-	var wg errgroup.Group
-	done := make(chan struct{})
-	var fixed []byte
-	var found bool
-
-	workers := runtime.GOMAXPROCS(0)
-	printer.S("         spinning up %d worker functions", runtime.GOMAXPROCS(0))
-	for i := 0; i < workers; i++ {
-		wg.Go(func() error {
-			// make a local copy of the buffer
-			buf := make([]byte, len(input))
-			copy(buf, input)
-
-			testFlip := func(idx int, pattern byte) bool {
-				// flip bits
-				buf[idx] ^= pattern
-
-				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
-				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
-				if err == nil {
-					printer.S("")
-					printer.S("        blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
-					printer.S("        hash is %v", restic.Hash(plaintext))
-					close(done)
-					found = true
-					fixed = plaintext
-					return true
-				}
-
-				// flip bits back
-				buf[idx] ^= pattern
-				return false
-			}
-
-			for i := range ch {
-				if bytewise {
-					for j := 0; j < 255; j++ {
-						if testFlip(i, byte(j)) {
-							return nil
-						}
-					}
-				} else {
-					for j := 0; j < 7; j++ {
-						// flip each bit once
-						if testFlip(i, (1 << uint(j))) {
-							return nil
-						}
-					}
-				}
-			}
-			return nil
-		})
-	}
-
-	wg.Go(func() error {
-		defer close(ch)
-
-		start := time.Now()
-		info := time.Now()
-		for i := range input {
-			select {
-			case ch <- i:
-			case <-done:
-				printer.S("     done after %v", time.Since(start))
-				return nil
-			}
-
-			if time.Since(info) > time.Second {
-				secs := time.Since(start).Seconds()
-				gps := float64(i) / secs
-				remaining := len(input) - i
-				eta := time.Duration(float64(remaining)/gps) * time.Second
-
-				printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
-					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
-				info = time.Now()
-			}
-		}
-		return nil
-	})
-	err := wg.Wait()
-	if err != nil {
-		panic("all go routines can only return nil")
-	}
-
-	if !found {
-		printer.S("\n        blob could not be repaired")
-	}
-	return fixed
-}
-
-func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
-	// strip signature at the end
-	l := len(buf)
-	nonce, ct := buf[:16], buf[16:l-16]
-	out := make([]byte, len(ct))
-
-	c, err := aes.NewCipher(k.EncryptionKey[:])
-	if err != nil {
-		panic(fmt.Sprintf("unable to create cipher: %v", err))
-	}
-	e := cipher.NewCTR(c, nonce)
-	e.XORKeyStream(out, ct)
-
-	return out
-}
-
-func loadBlobs(ctx context.Context, opts ExaminePackOptions, repo *Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error {
-	dec, err := zstd.NewReader(nil)
-	if err != nil {
-		panic(err)
-	}
-
-	packData, err := repo.LoadRaw(ctx, restic.PackFile, packID)
-	// allow processing broken pack files
-	if packData == nil {
-		return err
-	}
-
-	err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		for _, blob := range list {
-			printer.S("      loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
-			if int(blob.Offset+blob.Length) > len(packData) {
-				printer.E("skipping truncated blob")
-				continue
-			}
-			buf := packData[blob.Offset : blob.Offset+blob.Length]
-			key := repo.Key()
-
-			nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
-			plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
-			outputPrefix := ""
-			filePrefix := ""
-			if err != nil {
-				printer.E("error decrypting blob: %v", err)
-				if opts.TryRepair || opts.RepairByte {
-					plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
-				}
-				if plaintext != nil {
-					outputPrefix = "repaired "
-					filePrefix = "repaired-"
-				} else {
-					plaintext = decryptUnsigned(key, buf)
-					err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
-					if err != nil {
-						return err
-					}
-					continue
-				}
-			}
-
-			if blob.IsCompressed() {
-				decompressed, err := dec.DecodeAll(plaintext, nil)
-				if err != nil {
-					printer.S("         failed to decompress blob %v", blob.ID)
-				}
-				if decompressed != nil {
-					plaintext = decompressed
-				}
-			}
-
-			id := restic.Hash(plaintext)
-			var prefix string
-			if !id.Equal(blob.ID) {
-				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
-				prefix = "wrong-hash-"
-			} else {
-				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
-				prefix = "correct-"
-			}
-			if opts.ExtractPack {
-				err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
-				if err != nil {
-					return err
-				}
-			}
-			if opts.ReuploadBlobs {
-				_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
-				if err != nil {
-					return err
-				}
-				printer.S("         uploaded %v %v", blob.Type, id)
-			}
-		}
-		return nil
-	})
-	return err
-}
-
-func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
-	filename := fmt.Sprintf("%s%s.bin", prefix, id)
-	f, err := os.Create(filename)
-	if err != nil {
-		return err
-	}
-
-	_, err = f.Write(plain)
-	if err != nil {
-		_ = f.Close()
-		return err
-	}
-
-	err = f.Close()
-	if err != nil {
-		return err
-	}
-
-	printer.S("decrypt of blob %v stored at %v", id, filename)
-	return nil
-}

internal/repository/index/index_parallel_test.go

@@ -15,8 +15,7 @@ import (
 var repoFixture = filepath.Join("..", "testdata", "test-repo.tar.gz")
 
 func TestRepositoryForAllIndexes(t *testing.T) {
-	repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
+	repo, _ := repository.TestFromFixture(t, repoFixture)
-	defer cleanup()
 
 	expectedIndexIDs := restic.NewIDSet()
 	rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {

internal/repository/index/master_index.go

@@ -280,6 +280,7 @@ func (mi *MasterIndex) MergeFinalIndexes() error {
 }
 
 func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, err error) error) error {
+	defer p.Done()
 	indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile)
 	if err != nil {
 		return err
@@ -302,7 +303,6 @@ func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked,
 			return err
 		}
 		p.SetMax(numIndexFiles)
-		defer p.Done()
 	}
 
 	err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, err error) error {

internal/repository/index/master_index_test.go

@@ -694,62 +694,3 @@ func TestRewriteSplitPacks(t *testing.T) {
 	blobs := mi.Lookup(blobOther.BlobHandle)
 	rtest.Equals(t, nil, blobs)
 }
-
-// TestRewriteFullPacks checks that Rewrite drops a duplicate full index for the same
-// pack while keeping the other index files and blob lookups intact. Creates 3 indexes:
-// - indexA: contains packA
-// - indexB: contains packB
-// - indexC: contains packB
-// After the rewrite, indexC must be dropped. The other indexes must be kept.
-func TestRewriteFullPacks(t *testing.T) {
-	originalFull := index.Full
-	defer func() {
-		index.Full = originalFull
-	}()
-	index.Full = func(*index.Index) bool { return true }
-
-	repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion)
-
-	packA := restic.NewRandomID()
-	packB := restic.NewRandomID()
-
-	blobA := restic.PackedBlob{
-		PackID: packA,
-		Blob: restic.Blob{
-			BlobHandle: restic.NewRandomBlobHandle(),
-			Length:     uint(crypto.CiphertextLength(10)),
-			Offset:     0,
-		},
-	}
-	blobB := restic.PackedBlob{
-		PackID: packB,
-		Blob: restic.Blob{
-			BlobHandle: restic.NewRandomBlobHandle(),
-			Length:     uint(crypto.CiphertextLength(50)),
-			Offset:     0,
-		},
-	}
-
-	mi := index.NewMasterIndex()
-	rtest.OK(t, mi.StorePack(context.TODO(), packA, restic.Blobs{blobA.Blob}, unpacked))
-	rtest.OK(t, mi.Flush(context.TODO(), unpacked))
-	rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
-	rtest.OK(t, mi.Flush(context.TODO(), unpacked))
-	rtest.OK(t, mi.StorePack(context.TODO(), packB, restic.Blobs{blobB.Blob}, unpacked))
-	rtest.OK(t, mi.Flush(context.TODO(), unpacked))
-
-	indexIDs := mi.IDs()
-	rtest.Equals(t, 3, len(indexIDs))
-
-	rtest.OK(t, mi.Rewrite(context.TODO(), unpacked, nil, indexIDs, nil, index.MasterIndexRewriteOpts{}))
-
-	mi2 := index.NewMasterIndex()
-	rtest.OK(t, mi2.Load(context.TODO(), repo, nil, nil))
-
-	afterRewrite := mi2.IDs()
-	rtest.Equals(t, 2, len(afterRewrite))
-	rtest.Equals(t, 2, len(afterRewrite.Intersect(indexIDs)))
-
-	rtest.Equals(t, []restic.PackedBlob{blobA}, mi2.Lookup(blobA.BlobHandle))
-	rtest.Equals(t, []restic.PackedBlob{blobB}, mi2.Lookup(blobB.BlobHandle))
-}

internal/repository/index_list.go

@@ -1,41 +0,0 @@
-package repository
-
-import (
-	"context"
-	"iter"
-
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository/index"
-	"github.com/restic/restic/internal/restic"
-)
-
-// IndexBlob is one blob handle from an on-disk index file, or an error from loading/decoding
-// that file.
-type IndexBlob struct {
-	Handle restic.BlobHandle
-	Error  error
-}
-
-// AllIndexBlobs streams blob handles from each index file without building a master index.
-func AllIndexBlobs(ctx context.Context, lister restic.Lister, loader restic.LoaderUnpacked) iter.Seq[IndexBlob] {
-	return func(yield func(IndexBlob) bool) {
-		stopIteration := errors.New("stop index blob iteration")
-		err := index.ForAllIndexes(ctx, lister, loader, func(_ restic.ID, idx *index.Index, err error) error {
-			if err != nil {
-				return err
-			}
-			for blob := range idx.Values() {
-				if ctx.Err() != nil {
-					return ctx.Err()
-				}
-				if !yield(IndexBlob{Handle: blob.BlobHandle}) {
-					return stopIteration
-				}
-			}
-			return nil
-		})
-		if err != nil && !errors.Is(err, stopIteration) {
-			yield(IndexBlob{Error: err})
-		}
-	}
-}

internal/repository/index_list_test.go

@@ -1,62 +0,0 @@
-package repository_test
-
-import (
-	"context"
-	"testing"
-
-	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/restic"
-	rtest "github.com/restic/restic/internal/test"
-)
-
-func TestAllIndexBlobs(t *testing.T) {
-	repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
-
-	want := restic.NewBlobSet()
-	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		for i := range 5 {
-			data := []byte{byte('a' + i)}
-			id, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, data, restic.ID{}, false)
-			rtest.OK(t, err)
-			want.Insert(restic.BlobHandle{Type: restic.DataBlob, ID: id})
-		}
-		return nil
-	}))
-
-	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
-
-	fromMaster := restic.NewBlobSet()
-	rtest.OK(t, repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) {
-		fromMaster.Insert(pb.BlobHandle)
-	}))
-	rtest.Equals(t, want, fromMaster)
-
-	fromStream := restic.NewBlobSet()
-	for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
-		if entry.Error != nil {
-			t.Fatalf("unexpected error: %v", entry.Error)
-		}
-		fromStream.Insert(entry.Handle)
-	}
-	rtest.Equals(t, want, fromStream)
-}
-
-func TestAllIndexBlobsEarlyStop(t *testing.T) {
-	repo, _, _ := repository.TestRepositoryWithVersion(t, 0)
-
-	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-		for range 5 {
-			_, _, _, err := uploader.SaveBlob(ctx, restic.DataBlob, []byte("test"), restic.ID{}, false)
-			rtest.OK(t, err)
-		}
-		return nil
-	}))
-
-	var count int
-	for entry := range repository.AllIndexBlobs(context.TODO(), repo, repo) {
-		rtest.Assert(t, entry.Error == nil, "unexpected error after early stop: %v", entry.Error)
-		count++
-		break
-	}
-	rtest.Equals(t, 1, count)
-}

internal/repository/prune.go

@@ -586,6 +586,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
 	// unreferenced packs can be safely deleted first
 	if len(plan.removePacksFirst) != 0 {
 		printer.P("deleting unreferenced packs\n")
+		// ignoring errors is fine here as keeping too many packs cannot damage the repository
 		_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacksFirst, restic.PackFile, printer)
 		// forget unused data
 		plan.removePacksFirst = nil
@@ -643,6 +644,7 @@ func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) er
 
 	if len(plan.removePacks) != 0 {
 		printer.P("removing %d old packs", len(plan.removePacks))
+		// ignoring errors is fine here as keeping too many packs cannot damage the repository
 		_ = deleteFiles(ctx, true, &internalRepository{repo}, plan.removePacks, restic.PackFile, printer)
 	}
 	if ctx.Err() != nil {

internal/repository/repack_test.go

@@ -86,7 +86,7 @@ func selectBlobs(t *testing.T, random *rand.Rand, repo restic.Repository, p floa
 	blobs := restic.NewBlobSet()
 
 	err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
-		entries, err := repo.ListPack(context.TODO(), id, size)
+		entries, _, err := repo.ListPack(context.TODO(), id, size)
 		if err != nil {
 			t.Fatalf("error listing pack %v: %v", id, err)
 		}

internal/repository/repair_pack.go

@@ -76,8 +76,9 @@ func resolveBlobsForPacks(ctx context.Context, repo *Repository, ids restic.IDSe
 
 	err := repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		if ids.Has(id) {
-			blobs, err := repo.ListPack(ctx, id, size)
+			blobs, _, err := repo.ListPack(ctx, id, size)
 			if err != nil {
+				// ignore errors for broken pack files to be able to salvage as much as possible
 				return nil
 			}
 			packToBlobs[id] = blobs

internal/repository/repository.go

@@ -783,7 +783,7 @@ func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[rest
 	// a worker receives an pack ID from ch, reads the pack contents, and adds them to idx
 	worker := func() error {
 		for fi := range ch {
-			entries, err := r.ListPack(wgCtx, fi.ID, fi.Size)
+			entries, _, err := r.ListPack(wgCtx, fi.ID, fi.Size)
 			if err != nil {
 				debug.Log("unable to list pack file %v", fi.ID.Str())
 				m.Lock()
@@ -844,13 +844,8 @@ func (r *Repository) prepareCache() error {
 
 	packs := r.idx.Packs(restic.NewIDSet())
 
-	ids := make(map[string]struct{})
-	for id := range packs {
-		ids[id.String()] = struct{}{}
-	}
-
 	// clear old packs
-	return r.cache.Clear(backend.PackFile, ids)
+	return r.cache.Clear(restic.PackFile, packs)
 }
 
 // SearchKey finds a key with the supplied password, afterwards the config is
@@ -964,11 +959,12 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
 	})
 }
 
-// ListPack returns the list of blobs saved in the pack id.
+// ListPack returns the list of blobs saved in the pack id and the length of
-func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, error) {
+// the pack header.
+func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (restic.Blobs, uint32, error) {
 	h := backend.Handle{Type: restic.PackFile, Name: id.String()}
 
-	entries, _, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
+	entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
 	if err != nil {
 		if r.cache != nil {
 			// ignore error as there is not much we can do here
@@ -976,9 +972,9 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) (re
 		}
 
 		// retry on error
-		entries, _, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
+		entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size)
 	}
-	return entries, err
+	return entries, hdrSize, err
 }
 
 // Delete calls backend.Delete() if implemented, and returns an error

internal/repository/repository_test.go

@@ -319,8 +319,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
 var repoFixture = filepath.Join("testdata", "test-repo.tar.gz")
 
 func TestRepositoryLoadIndex(t *testing.T) {
-	repo, _, cleanup := repository.TestFromFixture(t, repoFixture)
+	repo, _ := repository.TestFromFixture(t, repoFixture)
-	defer cleanup()
 
 	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
 }
@@ -373,8 +372,7 @@ func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length
 }
 
 func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
-	repodir, cleanup := rtest.Env(t, repoFixture)
+	repodir := rtest.Env(t, repoFixture)
-	defer cleanup()
 
 	be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2}, t.Logf)
 	rtest.OK(t, err)
@@ -479,7 +477,7 @@ func TestListPack(t *testing.T) {
 		return nil
 	}))
 
-	blobs, err := repo.ListPack(context.TODO(), packID, size)
+	blobs, _, err := repo.ListPack(context.TODO(), packID, size)
 	rtest.OK(t, err)
 	rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs)
 

internal/repository/testing.go

@@ -105,11 +105,11 @@ func TestRepositoryWithVersion(t testing.TB, version uint) (*Repository, restic.
 	return repo, &internalRepository{repo}, be
 }
 
-func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend, func()) {
+func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend) {
-	repodir, cleanup := test.Env(t, repoFixture)
+	repodir := test.Env(t, repoFixture)
 	repo, be := TestOpenLocal(t, repodir)
 
-	return repo, be, cleanup
+	return repo, be
 }
 
 // TestOpenLocal opens a local repository.

internal/restic/backend_find.go

@@ -24,13 +24,13 @@ func (e *NoIDByPrefixError) Error() string {
 // Find loads the list of all files of type t and searches for names which
 // start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
 // If more than one is found, nil and ErrMultipleIDMatches is returned.
-func Find(ctx context.Context, repo Lister, t FileType, prefix string) (ID, error) {
+func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
 	match := ID{}
 
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	err := repo.List(ctx, t, func(id ID, _ int64) error {
+	err := be.List(ctx, t, func(id ID, _ int64) error {
 		name := id.String()
 		if len(name) >= len(prefix) && prefix == name[:len(prefix)] {
 			if match.IsNull() {

internal/restic/blob.go

@@ -57,6 +57,10 @@ func (h BlobHandle) String() string {
 	return fmt.Sprintf("<%s/%s>", h.Type, h.ID.Str())
 }
 
+func NewRandomBlobHandle() BlobHandle {
+	return BlobHandle{ID: NewRandomID(), Type: DataBlob}
+}
+
 // BlobType specifies what a blob stored in a pack is.
 type BlobType uint8
 

internal/restic/config.go

@@ -26,6 +26,11 @@ const MaxRepoVersion = 2
 // is newly created with Init().
 const StableRepoVersion = 2
 
+// JSONUnpackedLoader loads unpacked JSON.
+type JSONUnpackedLoader interface {
+	LoadJSONUnpacked(context.Context, FileType, ID, interface{}) error
+}
+
 // CreateConfig creates a config file with a randomly selected polynomial and
 // ID.
 func CreateConfig(version uint) (Config, error) {

internal/restic/id.go

@@ -1,9 +1,11 @@
 package restic
 
 import (
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"io"
 )
 
 // Hash returns the ID for data.
@@ -38,6 +40,17 @@ func (id ID) String() string {
 	return hex.EncodeToString(id[:])
 }
 
+// NewRandomID returns a randomly generated ID. When reading from rand fails,
+// the function panics.
+func NewRandomID() ID {
+	id := ID{}
+	_, err := io.ReadFull(rand.Reader, id[:])
+	if err != nil {
+		panic(err)
+	}
+	return id
+}
+
 const shortStr = 4
 
 // Str returns the shortened string version of id.

internal/restic/repository.go

@@ -32,8 +32,9 @@ type Repository interface {
 	// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
 	ListBlobs(ctx context.Context, fn func(PackedBlob)) error
 	ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs
-	// ListPack returns the list of blobs saved in the pack id.
+	// ListPack returns the list of blobs saved in the pack id and the length of
-	ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, err error)
+	// the pack header.
+	ListPack(ctx context.Context, id ID, packSize int64) (entries Blobs, hdrSize uint32, err error)
 
 	LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error)
 	LoadBlobsFromPack(ctx context.Context, packID ID, blobs Blobs, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error

internal/restic/testing.go

@@ -1,9 +1,7 @@
 package restic
 
 import (
-	"crypto/rand"
 	"fmt"
-	"io"
 )
 
 // TestParseID parses s as a ID and panics if that fails.
@@ -20,18 +18,3 @@ func TestParseID(s string) ID {
 func TestParseHandle(s string, t BlobType) BlobHandle {
 	return BlobHandle{ID: TestParseID(s), Type: t}
 }
-
-func NewRandomBlobHandle() BlobHandle {
-	return BlobHandle{ID: NewRandomID(), Type: DataBlob}
-}
-
-// NewRandomID returns a randomly generated ID. When reading from rand fails,
-// the function panics.
-func NewRandomID() ID {
-	id := ID{}
-	_, err := io.ReadFull(rand.Reader, id[:])
-	if err != nil {
-		panic(err)
-	}
-	return id
-}

internal/selfupdate/download.go

@@ -80,9 +80,12 @@ func extractToFile(buf []byte, filename, target string, printf func(string, ...i
 		return err
 	}
 	if err = newFile.Sync(); err != nil {
+		_ = newFile.Close()
+		_ = os.Remove(newFile.Name())
 		return err
 	}
 	if err = newFile.Close(); err != nil {
+		_ = os.Remove(newFile.Name())
 		return err
 	}
 

internal/selfupdate/github.go

@@ -87,6 +87,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
 			var msg githubError
 			jerr := json.NewDecoder(res.Body).Decode(&msg)
 			if jerr == nil {
+				_ = res.Body.Close()
 				return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n  %v", res.StatusCode, res.Status, msg.Message)
 			}
 		}
@@ -137,6 +138,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
 	}
 
 	if res.StatusCode != http.StatusOK {
+		_ = res.Body.Close()
 		return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
 	}
 

internal/test/helpers.go

@@ -137,10 +137,18 @@ func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
 
 // Env creates a test environment and extracts the repository fixture.
 // Returned is the repo path and a cleanup function.
-func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
+func Env(t testing.TB, repoFixture string) string {
 	t.Helper()
-	tempdir, err := os.MkdirTemp(TestTempDir, "restic-test-env-")
+
-	OK(t, err)
+	var tempdir string
+	if TestCleanupTempDirs {
+		tempdir = t.TempDir()
+	} else {
+		var err error
+		tempdir, err = os.MkdirTemp(TestTempDir, "restic-test-env-")
+		OK(t, err)
+		t.Logf("leaving temporary directory %v used for test", tempdir)
+	}
 
 	fd, err := os.Open(repoFixture)
 	if err != nil {
@@ -150,14 +158,7 @@ func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
 
 	SetupTarTestFixture(t, tempdir, repoFixture)
 
-	return filepath.Join(tempdir, "repo"), func() {
+	return filepath.Join(tempdir, "repo")
-		if !TestCleanupTempDirs {
-			t.Logf("leaving temporary directory %v used for test", tempdir)
-			return
-		}
-
-		RemoveAll(t, tempdir)
-	}
 }
 
 func isFile(fi os.FileInfo) bool {