Merge branch 'master' into windows-securitydesc

internal/archiver/archiver.go

@@ -8,10 +8,12 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
@@ -40,6 +42,18 @@ type ItemStats struct {
 	TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
 }
 
+type ChangeStats struct {
+	New       uint
+	Changed   uint
+	Unchanged uint
+}
+
+type Summary struct {
+	Files, Dirs    ChangeStats
+	ProcessedBytes uint64
+	ItemStats
+}
+
 // Add adds other to the current ItemStats.
 func (s *ItemStats) Add(other ItemStats) {
 	s.DataBlobs += other.DataBlobs
@@ -61,6 +75,8 @@ type Archiver struct {
 	blobSaver *BlobSaver
 	fileSaver *FileSaver
 	treeSaver *TreeSaver
+	mu        sync.Mutex
+	summary   *Summary
 
 	// Error is called for all errors that occur during backup.
 	Error ErrorFunc
@@ -182,12 +198,58 @@ func (arch *Archiver) error(item string, err error) error {
 	return errf
 }
 
+func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
+	arch.CompleteItem(item, previous, current, s, d)
+
+	arch.mu.Lock()
+	defer arch.mu.Unlock()
+
+	arch.summary.ItemStats.Add(s)
+
+	if current != nil {
+		arch.summary.ProcessedBytes += current.Size
+	} else {
+		// last item or an error occurred
+		return
+	}
+
+	switch current.Type {
+	case "dir":
+		switch {
+		case previous == nil:
+			arch.summary.Dirs.New++
+		case previous.Equals(*current):
+			arch.summary.Dirs.Unchanged++
+		default:
+			arch.summary.Dirs.Changed++
+		}
+
+	case "file":
+		switch {
+		case previous == nil:
+			arch.summary.Files.New++
+		case previous.Equals(*current):
+			arch.summary.Files.Unchanged++
+		default:
+			arch.summary.Files.Changed++
+		}
+	}
+}
+
 // nodeFromFileInfo returns the restic node from an os.FileInfo.
-func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
-	node, err := restic.NodeFromFileInfo(filename, fi)
+func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
+	node, err := restic.NodeFromFileInfo(filename, fi, ignoreXattrListError)
 	if !arch.WithAtime {
 		node.AccessTime = node.ModTime
 	}
+	if feature.Flag.Enabled(feature.DeviceIDForHardlinks) {
+		if node.Links == 1 || node.Type == "dir" {
+			// the DeviceID is only necessary for hardlinked files
+			// when using subvolumes or snapshots their deviceIDs tend to change which causes
+			// restic to upload new tree blobs
+			node.DeviceID = 0
+		}
+	}
 	// overwrite name to match that within the snapshot
 	node.Name = path.Base(snPath)
 	if err != nil {
@@ -222,12 +284,12 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
 	return err
 }
 
-// SaveDir stores a directory in the repo and returns the node. snPath is the
+// saveDir stores a directory in the repo and returns the node. snPath is the
 // path within the current snapshot.
-func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
+func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
 	debug.Log("%v %v", snPath, dir)
 
-	treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi)
+	treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false)
 	if err != nil {
 		return FutureNode{}, err
 	}
@@ -250,7 +312,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi
 		pathname := arch.FS.Join(dir, name)
 		oldNode := previous.Find(name)
 		snItem := join(snPath, name)
-		fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode)
+		fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
 
 		// return error early if possible
 		if err != nil {
@@ -318,6 +380,7 @@ func (fn *FutureNode) take(ctx context.Context) futureNodeResult {
 			return res
 		}
 	case <-ctx.Done():
+		return futureNodeResult{err: ctx.Err()}
 	}
 	return futureNodeResult{err: errors.Errorf("no result")}
 }
@@ -334,14 +397,14 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
 	return true
 }
 
-// Save saves a target (file or directory) to the repo. If the item is
+// save saves a target (file or directory) to the repo. If the item is
 // excluded, this function returns a nil node and error, with excluded set to
 // true.
 //
 // Errors and completion needs to be handled by the caller.
 //
 // snPath is the path within the current snapshot.
-func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
+func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
 	start := time.Now()
 
 	debug.Log("%v target %q, previous %v", snPath, target, previous)
@@ -380,9 +443,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 		if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) {
 			if arch.allBlobsPresent(previous) {
 				debug.Log("%v hasn't changed, using old list of blobs", target)
-				arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
+				arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
 				arch.CompleteBlob(previous.Size)
-				node, err := arch.nodeFromFileInfo(snPath, target, fi)
+				node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
 				if err != nil {
 					return FutureNode{}, false, err
 				}
@@ -445,9 +508,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 		fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
 			arch.StartFile(snPath)
 		}, func() {
-			arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0)
+			arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
 		}, func(node *restic.Node, stats ItemStats) {
-			arch.CompleteItem(snPath, previous, node, stats, time.Since(start))
+			arch.trackItem(snPath, previous, node, stats, time.Since(start))
 		})
 
 	case fi.IsDir():
@@ -462,9 +525,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 			return FutureNode{}, false, err
 		}
 
-		fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree,
+		fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
 			func(node *restic.Node, stats ItemStats) {
-				arch.CompleteItem(snItem, previous, node, stats, time.Since(start))
+				arch.trackItem(snItem, previous, node, stats, time.Since(start))
 			})
 		if err != nil {
 			debug.Log("SaveDir for %v returned error: %v", snPath, err)
@@ -478,7 +541,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 	default:
 		debug.Log("  %v other", target)
 
-		node, err := arch.nodeFromFileInfo(snPath, target, fi)
+		node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
 		if err != nil {
 			return FutureNode{}, false, err
 		}
@@ -545,9 +608,9 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
 	return fi, nil
 }
 
-// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path
+// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
 // within the current snapshot.
-func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
+func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
 
 	var node *restic.Node
 	if snPath != "/" {
@@ -561,7 +624,9 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
 		}
 
 		debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath)
-		node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi)
+		// in some cases reading xattrs for directories above the backup target is not allowed
+		// thus ignore errors for such folders.
+		node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi, true)
 		if err != nil {
 			return FutureNode{}, 0, err
 		}
@@ -585,7 +650,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
 
 		// this is a leaf node
 		if subatree.Leaf() {
-			fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
+			fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
 
 			if err != nil {
 				err = arch.error(subatree.Path, err)
@@ -619,8 +684,8 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
 		}
 
 		// not a leaf node, archive subtree
-		fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
-			arch.CompleteItem(snItem, oldNode, n, is, time.Since(start))
+		fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
+			arch.trackItem(snItem, oldNode, n, is, time.Since(start))
 		})
 		if err != nil {
 			return FutureNode{}, 0, err
@@ -688,6 +753,7 @@ type SnapshotOptions struct {
 	Tags           restic.TagList
 	Hostname       string
 	Excludes       []string
+	BackupStart    time.Time
 	Time           time.Time
 	ParentSnapshot *restic.Snapshot
 	ProgramVersion string
@@ -738,15 +804,17 @@ func (arch *Archiver) stopWorkers() {
 }
 
 // Snapshot saves several targets and returns a snapshot.
-func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) {
+func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) {
+	arch.summary = &Summary{}
+
 	cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	atree, err := NewTree(arch.FS, cleanTargets)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	var rootTreeID restic.ID
@@ -762,8 +830,8 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
 			arch.runWorkers(wgCtx, wg)
 
 			debug.Log("starting snapshot")
-			fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
-				arch.CompleteItem("/", nil, nil, is, time.Since(start))
+			fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
+				arch.trackItem("/", nil, nil, is, time.Since(start))
 			})
 			if err != nil {
 				return err
@@ -799,12 +867,12 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
 	})
 	err = wgUp.Wait()
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	sn.ProgramVersion = opts.ProgramVersion
@@ -813,11 +881,28 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
 		sn.Parent = opts.ParentSnapshot.ID()
 	}
 	sn.Tree = &rootTreeID
+	sn.Summary = &restic.SnapshotSummary{
+		BackupStart: opts.BackupStart,
+		BackupEnd:   time.Now(),
+
+		FilesNew:            arch.summary.Files.New,
+		FilesChanged:        arch.summary.Files.Changed,
+		FilesUnmodified:     arch.summary.Files.Unchanged,
+		DirsNew:             arch.summary.Dirs.New,
+		DirsChanged:         arch.summary.Dirs.Changed,
+		DirsUnmodified:      arch.summary.Dirs.Unchanged,
+		DataBlobs:           arch.summary.ItemStats.DataBlobs,
+		TreeBlobs:           arch.summary.ItemStats.TreeBlobs,
+		DataAdded:           arch.summary.ItemStats.DataSize + arch.summary.ItemStats.TreeSize,
+		DataAddedPacked:     arch.summary.ItemStats.DataSizeInRepo + arch.summary.ItemStats.TreeSizeInRepo,
+		TotalFilesProcessed: arch.summary.Files.New + arch.summary.Files.Changed + arch.summary.Files.Unchanged,
+		TotalBytesProcessed: arch.summary.ProcessedBytes,
+	}
 
 	id, err := restic.SaveSnapshot(ctx, arch.Repo, sn)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
-	return sn, id, nil
+	return sn, id, arch.summary, nil
 }

internal/archiver/archiver_test.go

@@ -19,15 +19,16 @@ import (
 	"github.com/restic/restic/internal/backend/mem"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 	"golang.org/x/sync/errgroup"
 )
 
 func prepareTempdirRepoSrc(t testing.TB, src TestDir) (string, restic.Repository) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 	repo := repository.TestRepository(t)
 
 	TestCreateFiles(t, tempdir, src)
@@ -132,7 +133,7 @@ func TestArchiverSaveFile(t *testing.T) {
 	var tests = []TestFile{
 		{Content: ""},
 		{Content: "foo"},
-		{Content: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Content: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, testfile := range tests {
@@ -165,7 +166,7 @@ func TestArchiverSaveFileReaderFS(t *testing.T) {
 		Data string
 	}{
 		{Data: "foo"},
-		{Data: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Data: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, test := range tests {
@@ -207,7 +208,7 @@ func TestArchiverSave(t *testing.T) {
 	var tests = []TestFile{
 		{Content: ""},
 		{Content: "foo"},
-		{Content: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Content: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, testfile := range tests {
@@ -226,8 +227,9 @@ func TestArchiverSave(t *testing.T) {
 				return err
 			}
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
-			node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil)
+			node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -275,7 +277,7 @@ func TestArchiverSaveReaderFS(t *testing.T) {
 		Data string
 	}{
 		{Data: "foo"},
-		{Data: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Data: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, test := range tests {
@@ -303,8 +305,9 @@ func TestArchiverSaveReaderFS(t *testing.T) {
 				return err
 			}
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
-			node, excluded, err := arch.Save(ctx, "/", filename, nil)
+			node, excluded, err := arch.save(ctx, "/", filename, nil)
 			t.Logf("Save returned %v %v", node, err)
 			if err != nil {
 				t.Fatal(err)
@@ -351,7 +354,7 @@ func TestArchiverSaveReaderFS(t *testing.T) {
 func BenchmarkArchiverSaveFileSmall(b *testing.B) {
 	const fileSize = 4 * 1024
 	d := TestDir{"file": TestFile{
-		Content: string(restictest.Random(23, fileSize)),
+		Content: string(rtest.Random(23, fileSize)),
 	}}
 
 	b.SetBytes(fileSize)
@@ -383,7 +386,7 @@ func BenchmarkArchiverSaveFileSmall(b *testing.B) {
 func BenchmarkArchiverSaveFileLarge(b *testing.B) {
 	const fileSize = 40*1024*1024 + 1287898
 	d := TestDir{"file": TestFile{
-		Content: string(restictest.Random(23, fileSize)),
+		Content: string(rtest.Random(23, fileSize)),
 	}}
 
 	b.SetBytes(fileSize)
@@ -459,14 +462,14 @@ func appendToFile(t testing.TB, filename string, data []byte) {
 }
 
 func TestArchiverSaveFileIncremental(t *testing.T) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 
 	repo := &blobCountingRepo{
 		Repository: repository.TestRepository(t),
 		saved:      make(map[restic.BlobHandle]uint),
 	}
 
-	data := restictest.Random(23, 512*1024+887898)
+	data := rtest.Random(23, 512*1024+887898)
 	testfile := filepath.Join(tempdir, "testfile")
 
 	for i := 0; i < 3; i++ {
@@ -509,12 +512,12 @@ func chmodTwice(t testing.TB, name string) {
 	// POSIX says that ctime is updated "even if the file status does not
 	// change", but let's make sure it does change, just in case.
 	err := os.Chmod(name, 0700)
-	restictest.OK(t, err)
+	rtest.OK(t, err)
 
 	sleep()
 
 	err = os.Chmod(name, 0600)
-	restictest.OK(t, err)
+	rtest.OK(t, err)
 }
 
 func lstat(t testing.TB, name string) os.FileInfo {
@@ -553,7 +556,7 @@ func rename(t testing.TB, oldname, newname string) {
 }
 
 func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node {
-	node, err := restic.NodeFromFileInfo(filename, fi)
+	node, err := restic.NodeFromFileInfo(filename, fi, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -673,7 +676,7 @@ func TestFileChanged(t *testing.T) {
 				t.Skip("don't run test on Windows")
 			}
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 
 			filename := filepath.Join(tempdir, "file")
 			content := defaultContent
@@ -709,7 +712,7 @@ func TestFileChanged(t *testing.T) {
 }
 
 func TestFilChangedSpecialCases(t *testing.T) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 
 	filename := filepath.Join(tempdir, "file")
 	content := []byte("foobar")
@@ -743,12 +746,12 @@ func TestArchiverSaveDir(t *testing.T) {
 	}{
 		{
 			src: TestDir{
-				"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
+				"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
 			},
 			target: ".",
 			want: TestDir{
 				"targetdir": TestDir{
-					"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
+					"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
 				},
 			},
 		},
@@ -758,8 +761,8 @@ func TestArchiverSaveDir(t *testing.T) {
 					"foo":        TestFile{Content: "foo"},
 					"emptyfile":  TestFile{Content: ""},
 					"bar":        TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"},
-					"largefile":  TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
-					"largerfile": TestFile{Content: string(restictest.Random(234, 5*1024*1024+5000))},
+					"largefile":  TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
+					"largerfile": TestFile{Content: string(rtest.Random(234, 5*1024*1024+5000))},
 				},
 			},
 			target: "targetdir",
@@ -831,13 +834,14 @@ func TestArchiverSaveDir(t *testing.T) {
 
 			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
 			chdir := tempdir
 			if test.chdir != "" {
 				chdir = filepath.Join(chdir, test.chdir)
 			}
 
-			back := restictest.Chdir(t, chdir)
+			back := rtest.Chdir(t, chdir)
 			defer back()
 
 			fi, err := fs.Lstat(test.target)
@@ -845,7 +849,7 @@ func TestArchiverSaveDir(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil)
+			ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -895,7 +899,7 @@ func TestArchiverSaveDir(t *testing.T) {
 }
 
 func TestArchiverSaveDirIncremental(t *testing.T) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 
 	repo := &blobCountingRepo{
 		Repository: repository.TestRepository(t),
@@ -912,13 +916,14 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
 
 		arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 		arch.runWorkers(ctx, wg)
+		arch.summary = &Summary{}
 
 		fi, err := fs.Lstat(tempdir)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil)
+		ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -982,9 +987,9 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
 
 // bothZeroOrNeither fails the test if only one of exp, act is zero.
 func bothZeroOrNeither(tb testing.TB, exp, act uint64) {
+	tb.Helper()
 	if (exp == 0 && act != 0) || (exp != 0 && act == 0) {
-		_, file, line, _ := runtime.Caller(1)
-		tb.Fatalf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
+		rtest.Equals(tb, exp, act)
 	}
 }
 
@@ -1004,7 +1009,7 @@ func TestArchiverSaveTree(t *testing.T) {
 		prepare func(t testing.TB)
 		targets []string
 		want    TestDir
-		stat    ItemStats
+		stat    Summary
 	}{
 		{
 			src: TestDir{
@@ -1014,7 +1019,12 @@ func TestArchiverSaveTree(t *testing.T) {
 			want: TestDir{
 				"targetfile": TestFile{Content: string("foobar")},
 			},
-			stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
+			stat: Summary{
+				ItemStats:      ItemStats{1, 6, 32 + 6, 0, 0, 0},
+				ProcessedBytes: 6,
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{0, 0, 0},
+			},
 		},
 		{
 			src: TestDir{
@@ -1026,7 +1036,12 @@ func TestArchiverSaveTree(t *testing.T) {
 				"targetfile":  TestFile{Content: string("foobar")},
 				"filesymlink": TestSymlink{Target: "targetfile"},
 			},
-			stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
+			stat: Summary{
+				ItemStats:      ItemStats{1, 6, 32 + 6, 0, 0, 0},
+				ProcessedBytes: 6,
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{0, 0, 0},
+			},
 		},
 		{
 			src: TestDir{
@@ -1046,7 +1061,12 @@ func TestArchiverSaveTree(t *testing.T) {
 					"symlink": TestSymlink{Target: "subdir"},
 				},
 			},
-			stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
+			stat: Summary{
+				ItemStats:      ItemStats{0, 0, 0, 1, 0x154, 0x16a},
+				ProcessedBytes: 0,
+				Files:          ChangeStats{0, 0, 0},
+				Dirs:           ChangeStats{1, 0, 0},
+			},
 		},
 		{
 			src: TestDir{
@@ -1070,7 +1090,12 @@ func TestArchiverSaveTree(t *testing.T) {
 					},
 				},
 			},
-			stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
+			stat: Summary{
+				ItemStats:      ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
+				ProcessedBytes: 6,
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{3, 0, 0},
+			},
 		},
 	}
 
@@ -1082,20 +1107,13 @@ func TestArchiverSaveTree(t *testing.T) {
 
 			arch := New(repo, testFS, Options{})
 
-			var stat ItemStats
-			lock := &sync.Mutex{}
-			arch.CompleteItem = func(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
-				lock.Lock()
-				defer lock.Unlock()
-				stat.Add(s)
-			}
-
 			wg, ctx := errgroup.WithContext(context.TODO())
 			repo.StartPackUploader(ctx, wg)
 
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			if test.prepare != nil {
@@ -1107,7 +1125,7 @@ func TestArchiverSaveTree(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil)
+			fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1134,11 +1152,15 @@ func TestArchiverSaveTree(t *testing.T) {
 				want = test.src
 			}
 			TestEnsureTree(context.TODO(), t, "/", repo, treeID, want)
+			stat := arch.summary
 			bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs))
 			bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs))
 			bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize)
 			bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo)
 			bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo)
+			rtest.Equals(t, test.stat.ProcessedBytes, stat.ProcessedBytes)
+			rtest.Equals(t, test.stat.Files, stat.Files)
+			rtest.Equals(t, test.stat.Dirs, stat.Dirs)
 		})
 	}
 }
@@ -1386,7 +1408,7 @@ func TestArchiverSnapshot(t *testing.T) {
 				chdir = filepath.Join(chdir, filepath.FromSlash(test.chdir))
 			}
 
-			back := restictest.Chdir(t, chdir)
+			back := rtest.Chdir(t, chdir)
 			defer back()
 
 			var targets []string
@@ -1395,7 +1417,7 @@ func TestArchiverSnapshot(t *testing.T) {
 			}
 
 			t.Logf("targets: %v", targets)
-			sn, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
+			sn, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1408,7 +1430,7 @@ func TestArchiverSnapshot(t *testing.T) {
 			}
 			TestEnsureSnapshot(t, repo, snapshotID, want)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 
 			// check that the snapshot contains the targets with absolute paths
 			for i, target := range sn.Paths {
@@ -1539,11 +1561,11 @@ func TestArchiverSnapshotSelect(t *testing.T) {
 			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.Select = test.selFn
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			targets := []string{"."}
-			_, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
+			_, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
 			if test.err != "" {
 				if err == nil {
 					t.Fatalf("expected error not found, got %v, wanted %q", err, test.err)
@@ -1568,7 +1590,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
 			}
 			TestEnsureSnapshot(t, repo, snapshotID, want)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 		})
 	}
 }
@@ -1616,17 +1638,85 @@ func (f MockFile) Read(p []byte) (int, error) {
 	return n, err
 }
 
+func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) {
+	rtest.Equals(t, stat.Files.New, sn.Summary.FilesNew)
+	rtest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged)
+	rtest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified)
+	rtest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew)
+	rtest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged)
+	rtest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified)
+	rtest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed)
+	rtest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed)
+	bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs))
+	bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs))
+	bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded))
+	bothZeroOrNeither(t, uint64(stat.DataSizeInRepo+stat.TreeSizeInRepo), uint64(sn.Summary.DataAddedPacked))
+}
+
 func TestArchiverParent(t *testing.T) {
 	var tests = []struct {
-		src  TestDir
-		read map[string]int // tracks number of times a file must have been read
+		src         TestDir
+		modify      func(path string)
+		statInitial Summary
+		statSecond  Summary
 	}{
 		{
 			src: TestDir{
-				"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
+				"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
 			},
-			read: map[string]int{
-				"targetfile": 1,
+			statInitial: Summary{
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{0, 0, 0},
+				ProcessedBytes: 2102152,
+				ItemStats:      ItemStats{3, 0x201593, 0x201632, 1, 0, 0},
+			},
+			statSecond: Summary{
+				Files:          ChangeStats{0, 0, 1},
+				Dirs:           ChangeStats{0, 0, 0},
+				ProcessedBytes: 2102152,
+			},
+		},
+		{
+			src: TestDir{
+				"targetDir": TestDir{
+					"targetfile":  TestFile{Content: string(rtest.Random(888, 1234))},
+					"targetfile2": TestFile{Content: string(rtest.Random(888, 1235))},
+				},
+			},
+			statInitial: Summary{
+				Files:          ChangeStats{2, 0, 0},
+				Dirs:           ChangeStats{1, 0, 0},
+				ProcessedBytes: 2469,
+				ItemStats:      ItemStats{2, 0xe1c, 0xcd9, 2, 0, 0},
+			},
+			statSecond: Summary{
+				Files:          ChangeStats{0, 0, 2},
+				Dirs:           ChangeStats{0, 0, 1},
+				ProcessedBytes: 2469,
+			},
+		},
+		{
+			src: TestDir{
+				"targetDir": TestDir{
+					"targetfile": TestFile{Content: string(rtest.Random(888, 1234))},
+				},
+				"targetfile2": TestFile{Content: string(rtest.Random(888, 1235))},
+			},
+			modify: func(path string) {
+				remove(t, filepath.Join(path, "targetDir", "targetfile"))
+				save(t, filepath.Join(path, "targetfile2"), []byte("foobar"))
+			},
+			statInitial: Summary{
+				Files:          ChangeStats{2, 0, 0},
+				Dirs:           ChangeStats{1, 0, 0},
+				ProcessedBytes: 2469,
+				ItemStats:      ItemStats{2, 0xe13, 0xcf8, 2, 0, 0},
+			},
+			statSecond: Summary{
+				Files:          ChangeStats{0, 1, 0},
+				Dirs:           ChangeStats{0, 1, 0},
+				ProcessedBytes: 6,
+				ItemStats:      ItemStats{1, 0x305, 0x233, 2, 0, 0},
 			},
 		},
 	}
@@ -1645,10 +1735,10 @@ func TestArchiverParent(t *testing.T) {
 
 			arch := New(repo, testFS, Options{})
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
-			firstSnapshot, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+			firstSnapshot, firstSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1673,38 +1763,38 @@ func TestArchiverParent(t *testing.T) {
 				}
 				return nil
 			})
+			rtest.Equals(t, test.statInitial.Files, summary.Files)
+			rtest.Equals(t, test.statInitial.Dirs, summary.Dirs)
+			rtest.Equals(t, test.statInitial.ProcessedBytes, summary.ProcessedBytes)
+			checkSnapshotStats(t, firstSnapshot, test.statInitial)
+
+			if test.modify != nil {
+				test.modify(tempdir)
+			}
 
 			opts := SnapshotOptions{
 				Time:           time.Now(),
 				ParentSnapshot: firstSnapshot,
 			}
-			_, secondSnapshotID, err := arch.Snapshot(ctx, []string{"."}, opts)
+			testFS.bytesRead = map[string]int{}
+			secondSnapshot, secondSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, opts)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			// check that all files still been read exactly once
-			TestWalkFiles(t, ".", test.src, func(filename string, item interface{}) error {
-				file, ok := item.(TestFile)
-				if !ok {
-					return nil
-				}
-
-				n, ok := testFS.bytesRead[filename]
-				if !ok {
-					t.Fatalf("file %v was not read at all", filename)
-				}
-
-				if n != len(file.Content) {
-					t.Fatalf("file %v: read %v bytes, wanted %v bytes", filename, n, len(file.Content))
-				}
-				return nil
-			})
+			if test.modify == nil {
+				// check that no files were read this time
+				rtest.Equals(t, map[string]int{}, testFS.bytesRead)
+			}
+			rtest.Equals(t, test.statSecond.Files, summary.Files)
+			rtest.Equals(t, test.statSecond.Dirs, summary.Dirs)
+			rtest.Equals(t, test.statSecond.ProcessedBytes, summary.ProcessedBytes)
+			checkSnapshotStats(t, secondSnapshot, test.statSecond)
 
 			t.Logf("second backup saved as %v", secondSnapshotID.Str())
 			t.Logf("testfs: %v", testFS)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 		})
 	}
 }
@@ -1804,7 +1894,7 @@ func TestArchiverErrorReporting(t *testing.T) {
 
 			tempdir, repo := prepareTempdirRepoSrc(t, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			if test.prepare != nil {
@@ -1814,7 +1904,7 @@ func TestArchiverErrorReporting(t *testing.T) {
 			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.Error = test.errFn
 
-			_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+			_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 			if test.mustError {
 				if err != nil {
 					t.Logf("found expected error (%v), skipping further checks", err)
@@ -1837,7 +1927,7 @@ func TestArchiverErrorReporting(t *testing.T) {
 			}
 			TestEnsureSnapshot(t, repo, snapshotID, want)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 		})
 	}
 }
@@ -1874,7 +1964,7 @@ func TestArchiverContextCanceled(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 	TestCreateFiles(t, tempdir, TestDir{
 		"targetfile": TestFile{Content: "foobar"},
 	})
@@ -1882,12 +1972,12 @@ func TestArchiverContextCanceled(t *testing.T) {
 	// Ensure that the archiver itself reports the canceled context and not just the backend
 	repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{})
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 
-	_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+	_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 
 	if err != nil {
 		t.Logf("found expected error (%v)", err)
@@ -1968,16 +2058,16 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
 		{
 			src: TestDir{
 				"dir": TestDir{
-					"file0": TestFile{Content: string(restictest.Random(0, 1024))},
-					"file1": TestFile{Content: string(restictest.Random(1, 1024))},
-					"file2": TestFile{Content: string(restictest.Random(2, 1024))},
-					"file3": TestFile{Content: string(restictest.Random(3, 1024))},
-					"file4": TestFile{Content: string(restictest.Random(4, 1024))},
-					"file5": TestFile{Content: string(restictest.Random(5, 1024))},
-					"file6": TestFile{Content: string(restictest.Random(6, 1024))},
-					"file7": TestFile{Content: string(restictest.Random(7, 1024))},
-					"file8": TestFile{Content: string(restictest.Random(8, 1024))},
-					"file9": TestFile{Content: string(restictest.Random(9, 1024))},
+					"file0": TestFile{Content: string(rtest.Random(0, 1024))},
+					"file1": TestFile{Content: string(rtest.Random(1, 1024))},
+					"file2": TestFile{Content: string(rtest.Random(2, 1024))},
+					"file3": TestFile{Content: string(rtest.Random(3, 1024))},
+					"file4": TestFile{Content: string(rtest.Random(4, 1024))},
+					"file5": TestFile{Content: string(rtest.Random(5, 1024))},
+					"file6": TestFile{Content: string(rtest.Random(6, 1024))},
+					"file7": TestFile{Content: string(rtest.Random(7, 1024))},
+					"file8": TestFile{Content: string(rtest.Random(8, 1024))},
+					"file9": TestFile{Content: string(rtest.Random(9, 1024))},
 				},
 			},
 			wantOpen: map[string]uint{
@@ -2002,7 +2092,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
 
 			tempdir, repo := prepareTempdirRepoSrc(t, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			testFS := &TrackFS{
@@ -2026,7 +2116,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
 				SaveBlobConcurrency: 1,
 			})
 
-			_, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+			_, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 			if !errors.Is(err, test.err) {
 				t.Errorf("expected error (%v) not found, got %v", test.err, err)
 			}
@@ -2054,7 +2144,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Sna
 		Time:           time.Now(),
 		ParentSnapshot: parent,
 	}
-	snapshot, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
+	snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2125,6 +2215,8 @@ const (
 )
 
 func TestMetadataChanged(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()
+
 	files := TestDir{
 		"testfile": TestFile{
 			Content: "foo bar test file",
@@ -2133,12 +2225,12 @@ func TestMetadataChanged(t *testing.T) {
 
 	tempdir, repo := prepareTempdirRepoSrc(t, files)
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	// get metadata
 	fi := lstat(t, "testfile")
-	want, err := restic.NodeFromFileInfo("testfile", fi)
+	want, err := restic.NodeFromFileInfo("testfile", fi, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2153,6 +2245,7 @@ func TestMetadataChanged(t *testing.T) {
 	sn, node2 := snapshot(t, repo, fs, nil, "testfile")
 
 	// set some values so we can then compare the nodes
+	want.DeviceID = 0
 	want.Content = node2.Content
 	want.Path = ""
 	if len(want.ExtendedAttributes) == 0 {
@@ -2195,7 +2288,7 @@ func TestMetadataChanged(t *testing.T) {
 	// make sure the content matches
 	TestEnsureFileContent(context.Background(), t, repo, "testfile", node3, files["testfile"].(TestFile))
 
-	checker.TestCheckRepo(t, repo)
+	checker.TestCheckRepo(t, repo, false)
 }
 
 func TestRacyFileSwap(t *testing.T) {
@@ -2207,7 +2300,7 @@ func TestRacyFileSwap(t *testing.T) {
 
 	tempdir, repo := prepareTempdirRepoSrc(t, files)
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	// get metadata of current folder
@@ -2236,7 +2329,7 @@ func TestRacyFileSwap(t *testing.T) {
 	arch.runWorkers(ctx, wg)
 
 	// fs.Track will panic if the file was not closed
-	_, excluded, err := arch.Save(ctx, "/", tempfile, nil)
+	_, excluded, err := arch.save(ctx, "/", tempfile, nil)
 	if err == nil {
 		t.Errorf("Save() should have failed")
 	}

internal/archiver/archiver_unix_test.go

@@ -6,6 +6,12 @@ package archiver
 import (
 	"os"
 	"syscall"
+	"testing"
+
+	"github.com/restic/restic/internal/feature"
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 type wrappedFileInfo struct {
@@ -39,3 +45,45 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo {
 
 	return res
 }
+
+func statAndSnapshot(t *testing.T, repo restic.Repository, name string) (*restic.Node, *restic.Node) {
+	fi := lstat(t, name)
+	want, err := restic.NodeFromFileInfo(name, fi, false)
+	rtest.OK(t, err)
+
+	_, node := snapshot(t, repo, fs.Local{}, nil, name)
+	return want, node
+}
+
+func TestHardlinkMetadata(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()
+
+	files := TestDir{
+		"testfile": TestFile{
+			Content: "foo bar test file",
+		},
+		"linktarget": TestFile{
+			Content: "test file",
+		},
+		"testlink": TestHardlink{
+			Target: "./linktarget",
+		},
+		"testdir": TestDir{},
+	}
+
+	tempdir, repo := prepareTempdirRepoSrc(t, files)
+
+	back := rtest.Chdir(t, tempdir)
+	defer back()
+
+	want, node := statAndSnapshot(t, repo, "testlink")
+	rtest.Assert(t, node.DeviceID == want.DeviceID, "device id mismatch expected %v got %v", want.DeviceID, node.DeviceID)
+	rtest.Assert(t, node.Links == want.Links, "link count mismatch expected %v got %v", want.Links, node.Links)
+	rtest.Assert(t, node.Inode == want.Inode, "inode mismatch expected %v got %v", want.Inode, node.Inode)
+
+	_, node = statAndSnapshot(t, repo, "testfile")
+	rtest.Assert(t, node.DeviceID == 0, "device id mismatch for testfile expected %v got %v", 0, node.DeviceID)
+
+	_, node = statAndSnapshot(t, repo, "testdir")
+	rtest.Assert(t, node.DeviceID == 0, "device id mismatch for testdir expected %v got %v", 0, node.DeviceID)
+}

internal/archiver/file_saver.go

@@ -29,7 +29,7 @@ type FileSaver struct {
 
 	CompleteBlob func(bytes uint64)
 
-	NodeFromFileInfo func(snPath, filename string, fi os.FileInfo) (*restic.Node, error)
+	NodeFromFileInfo func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error)
 }
 
 // NewFileSaver returns a new file saver. A worker pool with fileWorkers is
@@ -156,7 +156,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
 
 	debug.Log("%v", snPath)
 
-	node, err := s.NodeFromFileInfo(snPath, f.Name(), fi)
+	node, err := s.NodeFromFileInfo(snPath, f.Name(), fi, false)
 	if err != nil {
 		_ = f.Close()
 		completeError(err)

internal/archiver/file_saver_test.go

@@ -49,8 +49,8 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont
 	}
 
 	s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers)
-	s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
-		return restic.NodeFromFileInfo(filename, fi)
+	s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
+		return restic.NodeFromFileInfo(filename, fi, ignoreXattrListError)
 	}
 
 	return s, ctx, wg

internal/archiver/scanner_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/fs"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 func TestScanner(t *testing.T) {
@@ -81,10 +81,10 @@ func TestScanner(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			TestCreateFiles(t, tempdir, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			cur, err := os.Getwd()
@@ -216,10 +216,10 @@ func TestScannerError(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			TestCreateFiles(t, tempdir, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			cur, err := os.Getwd()
@@ -288,10 +288,10 @@ func TestScannerCancel(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 	TestCreateFiles(t, tempdir, src)
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	cur, err := os.Getwd()

internal/archiver/testing.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -31,7 +32,7 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res
 		}
 		opts.ParentSnapshot = sn
 	}
-	sn, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
+	sn, _, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -63,11 +64,29 @@ func (s TestSymlink) String() string {
 	return "<Symlink>"
 }
 
+// TestHardlink describes a hardlink created for a test.
+type TestHardlink struct {
+	Target string
+}
+
+func (s TestHardlink) String() string {
+	return "<Hardlink>"
+}
+
 // TestCreateFiles creates a directory structure described by dir at target,
 // which must already exist. On Windows, symlinks aren't created.
 func TestCreateFiles(t testing.TB, target string, dir TestDir) {
 	t.Helper()
-	for name, item := range dir {
+
+	// ensure a stable order such that it can be guaranteed that a hardlink target already exists
+	var names []string
+	for name := range dir {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		item := dir[name]
 		targetPath := filepath.Join(target, name)
 
 		switch it := item.(type) {
@@ -81,6 +100,11 @@ func TestCreateFiles(t testing.TB, target string, dir TestDir) {
 			if err != nil {
 				t.Fatal(err)
 			}
+		case TestHardlink:
+			err := fs.Link(filepath.Join(target, filepath.FromSlash(it.Target)), targetPath)
+			if err != nil {
+				t.Fatal(err)
+			}
 		case TestDir:
 			err := fs.Mkdir(targetPath, 0755)
 			if err != nil {

internal/archiver/testing_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 // MockT passes through all logging functions from T, but catches Fail(),
@@ -101,7 +101,7 @@ func TestTestCreateFiles(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		tempdir := restictest.TempDir(t)
+		tempdir := rtest.TempDir(t)
 
 		t.Run("", func(t *testing.T) {
 			tempdir := filepath.Join(tempdir, fmt.Sprintf("test-%d", i))
@@ -191,7 +191,7 @@ func TestTestWalkFiles(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 
 			got := make(map[string]string)
 
@@ -321,7 +321,7 @@ func TestTestEnsureFiles(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			createFilesAt(t, tempdir, test.files)
 
 			subtestT := testing.TB(t)
@@ -452,7 +452,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 
 			targetDir := filepath.Join(tempdir, "target")
 			err := fs.Mkdir(targetDir, 0700)
@@ -462,7 +462,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
 
 			createFilesAt(t, targetDir, test.files)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			repo := repository.TestRepository(t)
@@ -473,7 +473,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
 				Hostname: "localhost",
 				Tags:     []string{"test"},
 			}
-			_, id, err := arch.Snapshot(ctx, []string{"."}, opts)
+			_, id, _, err := arch.Snapshot(ctx, []string{"."}, opts)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/archiver/tree_saver.go

@@ -90,6 +90,10 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
 		// return the error if it wasn't ignored
 		if fnr.err != nil {
 			debug.Log("err for %v: %v", fnr.snPath, fnr.err)
+			if fnr.err == context.Canceled {
+				return nil, stats, fnr.err
+			}
+
 			fnr.err = s.errFn(fnr.target, fnr.err)
 			if fnr.err == nil {
 				// ignore error

internal/archiver/tree_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/fs"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 // debug.Log requires Tree.String.
@@ -439,10 +439,10 @@ func TestTree(t *testing.T) {
 				t.Skip("skip test on unix")
 			}
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			TestCreateFiles(t, tempdir, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			tree, err := NewTree(fs.Local{}, test.targets)

internal/backend/http_transport.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/peterbourgon/unixtransport"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 )
@@ -82,6 +83,8 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) {
 		TLSClientConfig:       &tls.Config{},
 	}
 
+	unixtransport.Register(tr)
+
 	if opts.InsecureTLS {
 		tr.TLSClientConfig.InsecureSkipVerify = true
 	}

internal/backend/layout/layout.go

@@ -10,6 +10,7 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 )
@@ -93,6 +94,8 @@ func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error
 // cannot be detected automatically.
 var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
 
+var ErrLegacyLayoutFound = errors.New("detected legacy S3 layout. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your repository")
+
 // DetectLayout tries to find out which layout is used in a local (or sftp)
 // filesystem at the given path. If repo is nil, an instance of LocalFilesystem
 // is used.
@@ -123,6 +126,10 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err
 	}
 
 	if foundKeyFile && !foundKeysFile {
+		if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
+			return nil, ErrLegacyLayoutFound
+		}
+
 		debug.Log("found s3 layout at %v", dir)
 		return &S3LegacyLayout{
 			Path: dir,
@@ -145,6 +152,10 @@ func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, pa
 			Join: repo.Join,
 		}
 	case "s3legacy":
+		if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
+			return nil, ErrLegacyLayoutFound
+		}
+
 		l = &S3LegacyLayout{
 			Path: path,
 			Join: repo.Join,

internal/backend/layout/layout_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/feature"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -352,6 +353,7 @@ func TestS3LegacyLayout(t *testing.T) {
 }
 
 func TestDetectLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {
@@ -389,6 +391,7 @@ func TestDetectLayout(t *testing.T) {
 }
 
 func TestParseLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {

internal/backend/local/config.go

@@ -10,7 +10,7 @@ import (
 // Config holds all information needed to open a local repository.
 type Config struct {
 	Path   string
-	Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"`
+	Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"`
 
 	Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"`
 }

internal/backend/local/layout_test.go

@@ -6,10 +6,12 @@ import (
 	"testing"
 
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/feature"
 	rtest "github.com/restic/restic/internal/test"
 )
 
 func TestLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {

internal/backend/mem/mem_backend.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"fmt"
 	"hash"
 	"io"
 	"net/http"
@@ -41,7 +42,7 @@ func NewFactory() location.Factory {
 	)
 }
 
-var errNotFound = errors.New("not found")
+var errNotFound = fmt.Errorf("not found")
 
 const connectionCount = 2
 

internal/backend/rest/config_test.go

@@ -31,6 +31,13 @@ var configTests = []test.ConfigTestData[Config]{
 			Connections: 5,
 		},
 	},
+	{
+		S: "rest:http+unix:///tmp/rest.socket:/my_backup_repo/",
+		Cfg: Config{
+			URL:         parseURL("http+unix:///tmp/rest.socket:/my_backup_repo/"),
+			Connections: 5,
+		},
+	},
 }
 
 func TestParseConfig(t *testing.T) {

internal/backend/rest/rest_test.go

@@ -1,11 +1,18 @@
+//go:build go1.20
+// +build go1.20
+
 package rest_test
 
 import (
+	"bufio"
 	"context"
-	"net"
+	"fmt"
 	"net/url"
 	"os"
 	"os/exec"
+	"regexp"
+	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -14,54 +21,133 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, func()) {
+var (
+	serverStartedRE = regexp.MustCompile("^start server on (.*)$")
+)
+
+func runRESTServer(ctx context.Context, t testing.TB, dir, reqListenAddr string) (*url.URL, func()) {
 	srv, err := exec.LookPath("rest-server")
 	if err != nil {
 		t.Skip(err)
 	}
 
-	cmd := exec.CommandContext(ctx, srv, "--no-auth", "--path", dir)
+	// create our own context, so that our cleanup can cancel and wait for completion
+	// this will ensure any open ports, open unix sockets etc are properly closed
+	processCtx, cancel := context.WithCancel(ctx)
+	cmd := exec.CommandContext(processCtx, srv, "--no-auth", "--path", dir, "--listen", reqListenAddr)
+
+	// this cancel func is called by when the process context is done
+	cmd.Cancel = func() error {
+		// we execute in a Go-routine as we know the caller will
+		// be waiting on a .Wait() regardless
+		go func() {
+			// try to send a graceful termination signal
+			if cmd.Process.Signal(syscall.SIGTERM) == nil {
+				// if we succeed, then wait a few seconds
+				time.Sleep(2 * time.Second)
+			}
+			// and then make sure it's killed either way, ignoring any error code
+			_ = cmd.Process.Kill()
+		}()
+		return nil
+	}
+
+	// this is the cleanup function that we return the caller,
+	// which will cancel our process context, and then wait for it to finish
+	cleanup := func() {
+		cancel()
+		_ = cmd.Wait()
+	}
+
+	// but in-case we don't finish this method, e.g. by calling t.Fatal()
+	// we also defer a call to clean it up ourselves, guarded by a flag to
+	// indicate that we returned the function to the caller to deal with.
+	callerWillCleanUp := false
+	defer func() {
+		if !callerWillCleanUp {
+			cleanup()
+		}
+	}()
+
+	// send stdout to our std out
 	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stdout
-	if err := cmd.Start(); err != nil {
-		t.Fatal(err)
-	}
 
-	// wait until the TCP port is reachable
-	var success bool
-	for i := 0; i < 10; i++ {
-		time.Sleep(200 * time.Millisecond)
-
-		c, err := net.Dial("tcp", "localhost:8000")
-		if err != nil {
-			continue
-		}
-
-		success = true
-		if err := c.Close(); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if !success {
-		t.Fatal("unable to connect to rest server")
-		return nil, nil
-	}
-
-	url, err := url.Parse("http://localhost:8000/restic-test/")
+	// capture stderr with a pipe, as we want to examine this output
+	// to determine when the server is started and listening.
+	cmdErr, err := cmd.StderrPipe()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	cleanup := func() {
-		if err := cmd.Process.Kill(); err != nil {
-			t.Fatal(err)
-		}
-
-		// ignore errors, we've killed the process
-		_ = cmd.Wait()
+	// start the rest-server
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
 	}
 
+	// create a channel to receive the actual listen address on
+	listenAddrCh := make(chan string)
+	go func() {
+		defer close(listenAddrCh)
+		matched := false
+		br := bufio.NewReader(cmdErr)
+		for {
+			line, err := br.ReadString('\n')
+			if err != nil {
+				// we ignore errors, as code that relies on this
+				// will happily fail via timeout and empty closed
+				// channel.
+				return
+			}
+
+			line = strings.Trim(line, "\r\n")
+			if !matched {
+				// look for the server started message, and return the address
+				// that it's listening on
+				matchedServerListen := serverStartedRE.FindSubmatch([]byte(line))
+				if len(matchedServerListen) == 2 {
+					listenAddrCh <- string(matchedServerListen[1])
+					matched = true
+				}
+			}
+			fmt.Fprintln(os.Stdout, line) // print all output to console
+		}
+	}()
+
+	// wait for us to get an address,
+	// or the parent context to cancel,
+	// or for us to timeout
+	var actualListenAddr string
+	select {
+	case <-processCtx.Done():
+		t.Fatal(context.Canceled)
+	case <-time.NewTimer(2 * time.Second).C:
+		t.Fatal(context.DeadlineExceeded)
+	case a, ok := <-listenAddrCh:
+		if !ok {
+			t.Fatal(context.Canceled)
+		}
+		actualListenAddr = a
+	}
+
+	// this translate the address that the server is listening on
+	// to a URL suitable for us to connect to
+	var addrToConnectTo string
+	if strings.HasPrefix(reqListenAddr, "unix:") {
+		addrToConnectTo = fmt.Sprintf("http+unix://%s:/restic-test/", actualListenAddr)
+	} else {
+		// while we may listen on 0.0.0.0, we connect to localhost
+		addrToConnectTo = fmt.Sprintf("http://%s/restic-test/", strings.Replace(actualListenAddr, "0.0.0.0", "localhost", 1))
+	}
+
+	// parse to a URL
+	url, err := url.Parse(addrToConnectTo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// indicate that we've completed successfully, and that the caller
+	// is responsible for calling cleanup
+	callerWillCleanUp = true
 	return url, cleanup
 }
 
@@ -91,7 +177,7 @@ func TestBackendREST(t *testing.T) {
 	defer cancel()
 
 	dir := rtest.TempDir(t)
-	serverURL, cleanup := runRESTServer(ctx, t, dir)
+	serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
 	defer cleanup()
 
 	newTestSuite(serverURL, false).RunTests(t)
@@ -116,7 +202,7 @@ func BenchmarkBackendREST(t *testing.B) {
 	defer cancel()
 
 	dir := rtest.TempDir(t)
-	serverURL, cleanup := runRESTServer(ctx, t, dir)
+	serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
 	defer cleanup()
 
 	newTestSuite(serverURL, false).RunBenchmarks(t)

internal/backend/rest/rest_unix_test.go

@@ -0,0 +1,30 @@
+//go:build !windows && go1.20
+// +build !windows,go1.20
+
+package rest_test
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestBackendRESTWithUnixSocket(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			rtest.SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
+		}
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dir := rtest.TempDir(t)
+	serverURL, cleanup := runRESTServer(ctx, t, path.Join(dir, "data"), fmt.Sprintf("unix:%s", path.Join(dir, "sock")))
+	defer cleanup()
+
+	newTestSuite(serverURL, false).RunTests(t)
+}

internal/backend/s3/config.go

@@ -20,7 +20,7 @@ type Config struct {
 	Secret       options.SecretString
 	Bucket       string
 	Prefix       string
-	Layout       string `option:"layout" help:"use this backend layout (default: auto-detect)"`
+	Layout       string `option:"layout" help:"use this backend layout (default: auto-detect) (deprecated)"`
 	StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
 
 	Connections   uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`

internal/backend/sftp/config.go

@@ -13,7 +13,7 @@ import (
 type Config struct {
 	User, Host, Port, Path string
 
-	Layout  string `option:"layout"  help:"use this backend directory layout (default: auto-detect)"`
+	Layout  string `option:"layout"  help:"use this backend directory layout (default: auto-detect) (deprecated)"`
 	Command string `option:"command" help:"specify command to create sftp connection"`
 	Args    string `option:"args"    help:"specify arguments for ssh"`
 

internal/backend/sftp/layout_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/sftp"
+	"github.com/restic/restic/internal/feature"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -16,6 +17,7 @@ func TestLayout(t *testing.T) {
 		t.Skip("sftp server binary not available")
 	}
 
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {

internal/cache/file.go

@@ -165,7 +165,8 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
 			continue
 		}
 
-		if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil {
+		// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
+		if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
 			return err
 		}
 	}

internal/checker/checker.go

@@ -106,9 +106,9 @@ func (c *Checker) LoadSnapshots(ctx context.Context) error {
 	return err
 }
 
-func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID]restic.BlobType {
+func computePackTypes(ctx context.Context, idx restic.MasterIndex) (map[restic.ID]restic.BlobType, error) {
 	packs := make(map[restic.ID]restic.BlobType)
-	idx.Each(ctx, func(pb restic.PackedBlob) {
+	err := idx.Each(ctx, func(pb restic.PackedBlob) {
 		tpe, exists := packs[pb.PackID]
 		if exists {
 			if pb.Type != tpe {
@@ -119,7 +119,7 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID
 		}
 		packs[pb.PackID] = tpe
 	})
-	return packs
+	return packs, err
 }
 
 // LoadIndex loads all index files.
@@ -169,7 +169,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 
 		debug.Log("process blobs")
 		cnt := 0
-		index.Each(ctx, func(blob restic.PackedBlob) {
+		err = index.Each(ctx, func(blob restic.PackedBlob) {
 			cnt++
 
 			if _, ok := packToIndex[blob.PackID]; !ok {
@@ -179,7 +179,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 		})
 
 		debug.Log("%d blobs processed", cnt)
-		return nil
+		return err
 	})
 	if err != nil {
 		errs = append(errs, err)
@@ -193,8 +193,14 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 	}
 
 	// compute pack size using index entries
-	c.packs = pack.Size(ctx, c.masterIndex, false)
-	packTypes := computePackTypes(ctx, c.masterIndex)
+	c.packs, err = pack.Size(ctx, c.masterIndex, false)
+	if err != nil {
+		return hints, append(errs, err)
+	}
+	packTypes, err := computePackTypes(ctx, c.masterIndex)
+	if err != nil {
+		return hints, append(errs, err)
+	}
 
 	debug.Log("checking for duplicate packs")
 	for packID := range c.packs {
@@ -484,7 +490,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
 }
 
 // UnusedBlobs returns all blobs that have never been referenced.
-func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
+func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, err error) {
 	if !c.trackUnused {
 		panic("only works when tracking blob references")
 	}
@@ -495,7 +501,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	c.repo.Index().Each(ctx, func(blob restic.PackedBlob) {
+	err = c.repo.Index().Each(ctx, func(blob restic.PackedBlob) {
 		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
 		if !c.blobRefs.M.Has(h) {
 			debug.Log("blob %v not referenced", h)
@@ -503,7 +509,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
 		}
 	})
 
-	return blobs
+	return blobs, err
 }
 
 // CountPacks returns the number of packs in the repository.

internal/checker/checker_test.go

@@ -72,11 +72,9 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) {
 }
 
 func TestCheckRepo(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
 	if len(errs) > 0 {
@@ -92,11 +90,9 @@ func TestCheckRepo(t *testing.T) {
 }
 
 func TestMissingPack(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	packHandle := backend.Handle{
 		Type: restic.PackFile,
 		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
@@ -123,11 +119,9 @@ func TestMissingPack(t *testing.T) {
 }
 
 func TestUnreferencedPack(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	// index 3f1a only references pack 60e0
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
 	indexHandle := backend.Handle{
@@ -156,11 +150,9 @@ func TestUnreferencedPack(t *testing.T) {
 }
 
 func TestUnreferencedBlobs(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	snapshotHandle := backend.Handle{
 		Type: restic.SnapshotFile,
 		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
@@ -188,18 +180,17 @@ func TestUnreferencedBlobs(t *testing.T) {
 	test.OKs(t, checkPacks(chkr))
 	test.OKs(t, checkStruct(chkr))
 
-	blobs := chkr.UnusedBlobs(context.TODO())
+	blobs, err := chkr.UnusedBlobs(context.TODO())
+	test.OK(t, err)
 	sort.Sort(blobs)
 
 	test.Equals(t, unusedBlobsBySnapshot, blobs)
 }
 
 func TestModifiedIndex(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	done := make(chan struct{})
 	defer close(done)
 
@@ -274,11 +265,9 @@ func TestModifiedIndex(t *testing.T) {
 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 
 func TestDuplicatePacksInIndex(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerDuplicateIndexTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
 	if len(hints) == 0 {
@@ -342,9 +331,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	t.Logf("archived as %v", sn.ID().Str())
 
 	beError := &errorBackend{Backend: repo.Backend()}
-	checkRepo, err := repository.New(beError, repository.Options{})
-	test.OK(t, err)
-	test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5, ""))
+	checkRepo := repository.TestOpenBackend(t, beError)
 
 	chkr := checker.New(checkRepo, false)
 
@@ -399,10 +386,8 @@ func (r *loadTreesOnceRepository) LoadTree(ctx context.Context, id restic.ID) (*
 }
 
 func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
-
-	repo := repository.TestOpenLocal(t, repodir)
 	checkRepo := &loadTreesOnceRepository{
 		Repository:  repo,
 		loadedTrees: restic.NewIDSet(),
@@ -549,9 +534,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
 }
 
 func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, func()) {
-	repodir, cleanup := test.Env(t, checkerTestData)
-
-	repo := repository.TestOpenLocal(t, repodir)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)

internal/checker/testing.go

@@ -8,7 +8,7 @@ import (
 )
 
 // TestCheckRepo runs the checker on repo.
-func TestCheckRepo(t testing.TB, repo restic.Repository) {
+func TestCheckRepo(t testing.TB, repo restic.Repository, skipStructure bool) {
 	chkr := New(repo, true)
 
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -33,18 +33,23 @@ func TestCheckRepo(t testing.TB, repo restic.Repository) {
 		t.Error(err)
 	}
 
-	// structure
-	errChan = make(chan error)
-	go chkr.Structure(context.TODO(), nil, errChan)
+	if !skipStructure {
+		// structure
+		errChan = make(chan error)
+		go chkr.Structure(context.TODO(), nil, errChan)
 
-	for err := range errChan {
-		t.Error(err)
-	}
+		for err := range errChan {
+			t.Error(err)
+		}
 
-	// unused blobs
-	blobs := chkr.UnusedBlobs(context.TODO())
-	if len(blobs) > 0 {
-		t.Errorf("unused blobs found: %v", blobs)
+		// unused blobs
+		blobs, err := chkr.UnusedBlobs(context.TODO())
+		if err != nil {
+			t.Error(err)
+		}
+		if len(blobs) > 0 {
+			t.Errorf("unused blobs found: %v", blobs)
+		}
 	}
 
 	// read data

internal/dump/common_test.go

@@ -78,7 +78,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
 			back := rtest.Chdir(t, tmpdir)
 			defer back()
 
-			sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
+			sn, _, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
 			rtest.OK(t, err)
 
 			tree, err := restic.LoadTree(ctx, repo, *sn.Tree)

internal/feature/features.go

@@ -0,0 +1,140 @@
+package feature
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type state string
+type FlagName string
+
+const (
+	// Alpha features are disabled by default. They do not guarantee any backwards compatibility and may change in arbitrary ways between restic versions.
+	Alpha state = "alpha"
+	// Beta features are enabled by default. They may still change, but incompatible changes should be avoided.
+	Beta state = "beta"
+	// Stable features are always enabled
+	Stable state = "stable"
+	// Deprecated features are always disabled
+	Deprecated state = "deprecated"
+)
+
+type FlagDesc struct {
+	Type        state
+	Description string
+}
+
+type FlagSet struct {
+	flags   map[FlagName]*FlagDesc
+	enabled map[FlagName]bool
+}
+
+func New() *FlagSet {
+	return &FlagSet{}
+}
+
+func getDefault(phase state) bool {
+	switch phase {
+	case Alpha, Deprecated:
+		return false
+	case Beta, Stable:
+		return true
+	default:
+		panic("unknown feature phase")
+	}
+}
+
+func (f *FlagSet) SetFlags(flags map[FlagName]FlagDesc) {
+	f.flags = map[FlagName]*FlagDesc{}
+	f.enabled = map[FlagName]bool{}
+
+	for name, flag := range flags {
+		fcopy := flag
+		f.flags[name] = &fcopy
+		f.enabled[name] = getDefault(fcopy.Type)
+	}
+}
+
+func (f *FlagSet) Apply(flags string, logWarning func(string)) error {
+	if flags == "" {
+		return nil
+	}
+
+	selection := make(map[string]bool)
+
+	for _, flag := range strings.Split(flags, ",") {
+		parts := strings.SplitN(flag, "=", 2)
+
+		name := parts[0]
+		value := "true"
+		if len(parts) == 2 {
+			value = parts[1]
+		}
+
+		isEnabled, err := strconv.ParseBool(value)
+		if err != nil {
+			return fmt.Errorf("failed to parse value %q for feature flag %v: %w", value, name, err)
+		}
+
+		selection[name] = isEnabled
+	}
+
+	for name, value := range selection {
+		fname := FlagName(name)
+		flag := f.flags[fname]
+		if flag == nil {
+			return fmt.Errorf("unknown feature flag %q", name)
+		}
+
+		switch flag.Type {
+		case Alpha, Beta:
+			f.enabled[fname] = value
+		case Stable:
+			logWarning(fmt.Sprintf("feature flag %q is always enabled and will be removed in a future release", fname))
+		case Deprecated:
+			logWarning(fmt.Sprintf("feature flag %q is always disabled and will be removed in a future release", fname))
+		default:
+			panic("unknown feature phase")
+		}
+	}
+
+	return nil
+}
+
+func (f *FlagSet) Enabled(name FlagName) bool {
+	isEnabled, ok := f.enabled[name]
+	if !ok {
+		panic(fmt.Sprintf("unknown feature flag %v", name))
+	}
+
+	return isEnabled
+}
+
+// Help contains information about a feature.
+type Help struct {
+	Name        string
+	Type        string
+	Default     bool
+	Description string
+}
+
+func (f *FlagSet) List() []Help {
+	var help []Help
+
+	for name, flag := range f.flags {
+		help = append(help, Help{
+			Name:        string(name),
+			Type:        string(flag.Type),
+			Default:     getDefault(flag.Type),
+			Description: flag.Description,
+		})
+	}
+
+	sort.Slice(help, func(i, j int) bool {
+		return strings.Compare(help[i].Name, help[j].Name) < 0
+	})
+
+	return help
+}

internal/feature/features_test.go

@@ -0,0 +1,151 @@
+package feature_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/restic/restic/internal/feature"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+var (
+	alpha      = feature.FlagName("alpha-feature")
+	beta       = feature.FlagName("beta-feature")
+	stable     = feature.FlagName("stable-feature")
+	deprecated = feature.FlagName("deprecated-feature")
+)
+
+var testFlags = map[feature.FlagName]feature.FlagDesc{
+	alpha: {
+		Type:        feature.Alpha,
+		Description: "alpha",
+	},
+	beta: {
+		Type:        feature.Beta,
+		Description: "beta",
+	},
+	stable: {
+		Type:        feature.Stable,
+		Description: "stable",
+	},
+	deprecated: {
+		Type:        feature.Deprecated,
+		Description: "deprecated",
+	},
+}
+
+func buildTestFlagSet() *feature.FlagSet {
+	flags := feature.New()
+	flags.SetFlags(testFlags)
+	return flags
+}
+
+func TestFeatureDefaults(t *testing.T) {
+	flags := buildTestFlagSet()
+	for _, exp := range []struct {
+		flag  feature.FlagName
+		value bool
+	}{
+		{alpha, false},
+		{beta, true},
+		{stable, true},
+		{deprecated, false},
+	} {
+		rtest.Assert(t, flags.Enabled(exp.flag) == exp.value, "expected flag %v to have value %v got %v", exp.flag, exp.value, flags.Enabled(exp.flag))
+	}
+}
+
+func panicIfCalled(msg string) {
+	panic(msg)
+}
+
+func TestEmptyApply(t *testing.T) {
+	flags := buildTestFlagSet()
+	rtest.OK(t, flags.Apply("", panicIfCalled))
+
+	rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled")
+	rtest.Assert(t, flags.Enabled(beta), "expected beta feature to be enabled")
+}
+
+func TestFeatureApply(t *testing.T) {
+	flags := buildTestFlagSet()
+	rtest.OK(t, flags.Apply(string(alpha), panicIfCalled))
+	rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled")
+
+	rtest.OK(t, flags.Apply(fmt.Sprintf("%s=false", alpha), panicIfCalled))
+	rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled")
+
+	rtest.OK(t, flags.Apply(fmt.Sprintf("%s=true", alpha), panicIfCalled))
+	rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled again")
+
+	rtest.OK(t, flags.Apply(fmt.Sprintf("%s=false", beta), panicIfCalled))
+	rtest.Assert(t, !flags.Enabled(beta), "expected beta feature to be disabled")
+
+	logMsg := ""
+	log := func(msg string) {
+		logMsg = msg
+	}
+
+	rtest.OK(t, flags.Apply(fmt.Sprintf("%s=false", stable), log))
+	rtest.Assert(t, flags.Enabled(stable), "expected stable feature to remain enabled")
+	rtest.Assert(t, strings.Contains(logMsg, string(stable)), "unexpected log message for stable flag: %v", logMsg)
+
+	logMsg = ""
+	rtest.OK(t, flags.Apply(fmt.Sprintf("%s=true", deprecated), log))
+	rtest.Assert(t, !flags.Enabled(deprecated), "expected deprecated feature to remain disabled")
+	rtest.Assert(t, strings.Contains(logMsg, string(deprecated)), "unexpected log message for deprecated flag: %v", logMsg)
+}
+
+func TestFeatureMultipleApply(t *testing.T) {
+	flags := buildTestFlagSet()
+
+	rtest.OK(t, flags.Apply(fmt.Sprintf("%s=true,%s=false", alpha, beta), panicIfCalled))
+	rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled")
+	rtest.Assert(t, !flags.Enabled(beta), "expected beta feature to be disabled")
+}
+
+func TestFeatureApplyInvalid(t *testing.T) {
+	flags := buildTestFlagSet()
+
+	err := flags.Apply("invalid-flag", panicIfCalled)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "unknown feature flag"), "expected unknown feature flag error, got: %v", err)
+
+	err = flags.Apply(fmt.Sprintf("%v=invalid", alpha), panicIfCalled)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "failed to parse value"), "expected parsing error, got: %v", err)
+}
+
+func assertPanic(t *testing.T) {
+	if r := recover(); r == nil {
+		t.Fatal("should have panicked")
+	}
+}
+
+func TestFeatureQueryInvalid(t *testing.T) {
+	defer assertPanic(t)
+
+	flags := buildTestFlagSet()
+	flags.Enabled("invalid-flag")
+}
+
+func TestFeatureSetInvalidPhase(t *testing.T) {
+	defer assertPanic(t)
+
+	flags := feature.New()
+	flags.SetFlags(map[feature.FlagName]feature.FlagDesc{
+		"invalid": {
+			Type: "invalid",
+		},
+	})
+}
+
+func TestFeatureList(t *testing.T) {
+	flags := buildTestFlagSet()
+
+	rtest.Equals(t, []feature.Help{
+		{string(alpha), string(feature.Alpha), false, "alpha"},
+		{string(beta), string(feature.Beta), true, "beta"},
+		{string(deprecated), string(feature.Deprecated), false, "deprecated"},
+		{string(stable), string(feature.Stable), true, "stable"},
+	}, flags.List())
+}

internal/feature/registry.go

@@ -0,0 +1,19 @@
+package feature
+
+// Flag is named such that checking for a feature uses `feature.Flag.Enabled(feature.ExampleFeature)`.
+var Flag = New()
+
+// flag names are written in kebab-case
+const (
+	DeprecateLegacyIndex    FlagName = "deprecate-legacy-index"
+	DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout"
+	DeviceIDForHardlinks    FlagName = "device-id-for-hardlinks"
+)
+
+func init() {
+	Flag.SetFlags(map[FlagName]FlagDesc{
+		DeprecateLegacyIndex:    {Type: Beta, Description: "disable support for index format used by restic 0.1.0. Use `restic repair index` to update the index if necessary."},
+		DeprecateS3LegacyLayout: {Type: Beta, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your S3 repository if necessary."},
+		DeviceIDForHardlinks:    {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"},
+	})
+}

internal/feature/testing.go

@@ -0,0 +1,33 @@
+package feature
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestSetFlag temporarily sets a feature flag to the given value until the
+// returned function is called.
+//
+// Usage
+// ```
+// defer TestSetFlag(t, features.Flags, features.ExampleFlag, true)()
+// ```
+func TestSetFlag(t *testing.T, f *FlagSet, flag FlagName, value bool) func() {
+	current := f.Enabled(flag)
+
+	panicIfCalled := func(msg string) {
+		panic(msg)
+	}
+
+	if err := f.Apply(fmt.Sprintf("%s=%v", flag, value), panicIfCalled); err != nil {
+		// not reachable
+		panic(err)
+	}
+
+	return func() {
+		if err := f.Apply(fmt.Sprintf("%s=%v", flag, current), panicIfCalled); err != nil {
+			// not reachable
+			panic(err)
+		}
+	}
+}

internal/feature/testing_test.go

@@ -0,0 +1,19 @@
+package feature_test
+
+import (
+	"testing"
+
+	"github.com/restic/restic/internal/feature"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestSetFeatureFlag(t *testing.T) {
+	flags := buildTestFlagSet()
+	rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled")
+
+	restore := feature.TestSetFlag(t, flags, alpha, true)
+	rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled")
+
+	restore()
+	rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled again")
+}

internal/fs/fs_local_vss.go

@@ -3,41 +3,108 @@ package fs
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/options"
 )
 
-// ErrorHandler is used to report errors via callback
-type ErrorHandler func(item string, err error) error
+// VSSConfig holds extended options of windows volume shadow copy service.
+type VSSConfig struct {
+	ExcludeAllMountPoints bool          `option:"exclude-all-mount-points" help:"exclude mountpoints from snapshotting on all volumes"`
+	ExcludeVolumes        string        `option:"exclude-volumes" help:"semicolon separated list of volumes to exclude from snapshotting (ex. 'c:\\;e:\\mnt;\\\\?\\Volume{...}')"`
+	Timeout               time.Duration `option:"timeout" help:"time that the VSS can spend creating snapshot before timing out"`
+	Provider              string        `option:"provider" help:"VSS provider identifier which will be used for snapshotting"`
+}
+
+func init() {
+	if runtime.GOOS == "windows" {
+		options.Register("vss", VSSConfig{})
+	}
+}
+
+// NewVSSConfig returns a new VSSConfig with the default values filled in.
+func NewVSSConfig() VSSConfig {
+	return VSSConfig{
+		Timeout: time.Second * 120,
+	}
+}
+
+// ParseVSSConfig parses a VSS extended options to VSSConfig struct.
+func ParseVSSConfig(o options.Options) (VSSConfig, error) {
+	cfg := NewVSSConfig()
+	o = o.Extract("vss")
+	if err := o.Apply("vss", &cfg); err != nil {
+		return VSSConfig{}, err
+	}
+
+	return cfg, nil
+}
+
+// ErrorHandler is used to report errors via callback.
+type ErrorHandler func(item string, err error)
 
 // MessageHandler is used to report errors/messages via callbacks.
 type MessageHandler func(msg string, args ...interface{})
 
+// VolumeFilter is used to filter volumes by it's mount point or GUID path.
+type VolumeFilter func(volume string) bool
+
 // LocalVss is a wrapper around the local file system which uses windows volume
 // shadow copy service (VSS) in a transparent way.
 type LocalVss struct {
 	FS
-	snapshots       map[string]VssSnapshot
-	failedSnapshots map[string]struct{}
-	mutex           sync.RWMutex
-	msgError        ErrorHandler
-	msgMessage      MessageHandler
+	snapshots             map[string]VssSnapshot
+	failedSnapshots       map[string]struct{}
+	mutex                 sync.RWMutex
+	msgError              ErrorHandler
+	msgMessage            MessageHandler
+	excludeAllMountPoints bool
+	excludeVolumes        map[string]struct{}
+	timeout               time.Duration
+	provider              string
 }
 
 // statically ensure that LocalVss implements FS.
 var _ FS = &LocalVss{}
 
+// parseMountPoints try to convert semicolon separated list of mount points
+// to map of lowercased volume GUID pathes. Mountpoints already in volume
+// GUID path format will be validated and normalized.
+func parseMountPoints(list string, msgError ErrorHandler) (volumes map[string]struct{}) {
+	if list == "" {
+		return
+	}
+	for _, s := range strings.Split(list, ";") {
+		if v, err := GetVolumeNameForVolumeMountPoint(s); err != nil {
+			msgError(s, errors.Errorf("failed to parse vss.exclude-volumes [%s]: %s", s, err))
+		} else {
+			if volumes == nil {
+				volumes = make(map[string]struct{})
+			}
+			volumes[strings.ToLower(v)] = struct{}{}
+		}
+	}
+
+	return
+}
+
 // NewLocalVss creates a new wrapper around the windows filesystem using volume
 // shadow copy service to access locked files.
-func NewLocalVss(msgError ErrorHandler, msgMessage MessageHandler) *LocalVss {
+func NewLocalVss(msgError ErrorHandler, msgMessage MessageHandler, cfg VSSConfig) *LocalVss {
 	return &LocalVss{
-		FS:              Local{},
-		snapshots:       make(map[string]VssSnapshot),
-		failedSnapshots: make(map[string]struct{}),
-		msgError:        msgError,
-		msgMessage:      msgMessage,
+		FS:                    Local{},
+		snapshots:             make(map[string]VssSnapshot),
+		failedSnapshots:       make(map[string]struct{}),
+		msgError:              msgError,
+		msgMessage:            msgMessage,
+		excludeAllMountPoints: cfg.ExcludeAllMountPoints,
+		excludeVolumes:        parseMountPoints(cfg.ExcludeVolumes, msgError),
+		timeout:               cfg.Timeout,
+		provider:              cfg.Provider,
 	}
 }
 
@@ -50,7 +117,7 @@ func (fs *LocalVss) DeleteSnapshots() {
 
 	for volumeName, snapshot := range fs.snapshots {
 		if err := snapshot.Delete(); err != nil {
-			_ = fs.msgError(volumeName, errors.Errorf("failed to delete VSS snapshot: %s", err))
+			fs.msgError(volumeName, errors.Errorf("failed to delete VSS snapshot: %s", err))
 			activeSnapshots[volumeName] = snapshot
 		}
 	}
@@ -78,12 +145,27 @@ func (fs *LocalVss) Lstat(name string) (os.FileInfo, error) {
 	return os.Lstat(fs.snapshotPath(name))
 }
 
+// isMountPointIncluded  is true if given mountpoint included by user.
+func (fs *LocalVss) isMountPointIncluded(mountPoint string) bool {
+	if fs.excludeVolumes == nil {
+		return true
+	}
+
+	volume, err := GetVolumeNameForVolumeMountPoint(mountPoint)
+	if err != nil {
+		fs.msgError(mountPoint, errors.Errorf("failed to get volume from mount point [%s]: %s", mountPoint, err))
+		return true
+	}
+
+	_, ok := fs.excludeVolumes[strings.ToLower(volume)]
+	return !ok
+}
+
 // snapshotPath returns the path inside a VSS snapshots if it already exists.
 // If the path is not yet available as a snapshot, a snapshot is created.
 // If creation of a snapshot fails the file's original path is returned as
 // a fallback.
 func (fs *LocalVss) snapshotPath(path string) string {
-
 	fixPath := fixpath(path)
 
 	if strings.HasPrefix(fixPath, `\\?\UNC\`) {
@@ -114,23 +196,36 @@ func (fs *LocalVss) snapshotPath(path string) string {
 
 		if !snapshotExists && !snapshotFailed {
 			vssVolume := volumeNameLower + string(filepath.Separator)
-			fs.msgMessage("creating VSS snapshot for [%s]\n", vssVolume)
 
-			if snapshot, err := NewVssSnapshot(vssVolume, 120, fs.msgError); err != nil {
-				_ = fs.msgError(vssVolume, errors.Errorf("failed to create snapshot for [%s]: %s",
-					vssVolume, err))
+			if !fs.isMountPointIncluded(vssVolume) {
+				fs.msgMessage("snapshots for [%s] excluded by user\n", vssVolume)
 				fs.failedSnapshots[volumeNameLower] = struct{}{}
 			} else {
-				fs.snapshots[volumeNameLower] = snapshot
-				fs.msgMessage("successfully created snapshot for [%s]\n", vssVolume)
-				if len(snapshot.mountPointInfo) > 0 {
-					fs.msgMessage("mountpoints in snapshot volume [%s]:\n", vssVolume)
-					for mp, mpInfo := range snapshot.mountPointInfo {
-						info := ""
-						if !mpInfo.IsSnapshotted() {
-							info = " (not snapshotted)"
+				fs.msgMessage("creating VSS snapshot for [%s]\n", vssVolume)
+
+				var includeVolume VolumeFilter
+				if !fs.excludeAllMountPoints {
+					includeVolume = func(volume string) bool {
+						return fs.isMountPointIncluded(volume)
+					}
+				}
+
+				if snapshot, err := NewVssSnapshot(fs.provider, vssVolume, fs.timeout, includeVolume, fs.msgError); err != nil {
+					fs.msgError(vssVolume, errors.Errorf("failed to create snapshot for [%s]: %s",
+						vssVolume, err))
+					fs.failedSnapshots[volumeNameLower] = struct{}{}
+				} else {
+					fs.snapshots[volumeNameLower] = snapshot
+					fs.msgMessage("successfully created snapshot for [%s]\n", vssVolume)
+					if len(snapshot.mountPointInfo) > 0 {
+						fs.msgMessage("mountpoints in snapshot volume [%s]:\n", vssVolume)
+						for mp, mpInfo := range snapshot.mountPointInfo {
+							info := ""
+							if !mpInfo.IsSnapshotted() {
+								info = " (not snapshotted)"
+							}
+							fs.msgMessage(" - %s%s\n", mp, info)
 						}
-						fs.msgMessage(" - %s%s\n", mp, info)
 					}
 				}
 			}
@@ -173,9 +268,8 @@ func (fs *LocalVss) snapshotPath(path string) string {
 		snapshotPath = fs.Join(snapshot.GetSnapshotDeviceObject(),
 			strings.TrimPrefix(fixPath, volumeName))
 		if snapshotPath == snapshot.GetSnapshotDeviceObject() {
-			snapshotPath = snapshotPath + string(filepath.Separator)
+			snapshotPath += string(filepath.Separator)
 		}
-
 	} else {
 		// no snapshot is available for the requested path:
 		//  -> try to backup without a snapshot

internal/fs/fs_local_vss_test.go

@@ -0,0 +1,285 @@
+// +build windows
+
+package fs
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	ole "github.com/go-ole/go-ole"
+	"github.com/restic/restic/internal/options"
+)
+
+func matchStrings(ptrs []string, strs []string) bool {
+	if len(ptrs) != len(strs) {
+		return false
+	}
+
+	for i, p := range ptrs {
+		if p == "" {
+			return false
+		}
+		matched, err := regexp.MatchString(p, strs[i])
+		if err != nil {
+			panic(err)
+		}
+		if !matched {
+			return false
+		}
+	}
+
+	return true
+}
+
+func matchMap(strs []string, m map[string]struct{}) bool {
+	if len(strs) != len(m) {
+		return false
+	}
+
+	for _, s := range strs {
+		if _, ok := m[s]; !ok {
+			return false
+		}
+	}
+
+	return true
+}
+
+func TestVSSConfig(t *testing.T) {
+	type config struct {
+		excludeAllMountPoints bool
+		timeout               time.Duration
+		provider              string
+	}
+	setTests := []struct {
+		input  options.Options
+		output config
+	}{
+		{
+			options.Options{
+				"vss.timeout":  "6h38m42s",
+				"vss.provider": "Ms",
+			},
+			config{
+				timeout:  23922000000000,
+				provider: "Ms",
+			},
+		},
+		{
+			options.Options{
+				"vss.exclude-all-mount-points": "t",
+				"vss.provider":                 "{b5946137-7b9f-4925-af80-51abd60b20d5}",
+			},
+			config{
+				excludeAllMountPoints: true,
+				timeout:               120000000000,
+				provider:              "{b5946137-7b9f-4925-af80-51abd60b20d5}",
+			},
+		},
+		{
+			options.Options{
+				"vss.exclude-all-mount-points": "0",
+				"vss.exclude-volumes":          "",
+				"vss.timeout":                  "120s",
+				"vss.provider":                 "Microsoft Software Shadow Copy provider 1.0",
+			},
+			config{
+				timeout:  120000000000,
+				provider: "Microsoft Software Shadow Copy provider 1.0",
+			},
+		},
+	}
+	for i, test := range setTests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			cfg, err := ParseVSSConfig(test.input)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			errorHandler := func(item string, err error) {
+				t.Fatalf("unexpected error (%v)", err)
+			}
+			messageHandler := func(msg string, args ...interface{}) {
+				t.Fatalf("unexpected message (%s)", fmt.Sprintf(msg, args))
+			}
+
+			dst := NewLocalVss(errorHandler, messageHandler, cfg)
+
+			if dst.excludeAllMountPoints != test.output.excludeAllMountPoints ||
+				dst.excludeVolumes != nil || dst.timeout != test.output.timeout ||
+				dst.provider != test.output.provider {
+				t.Fatalf("wrong result, want:\n  %#v\ngot:\n  %#v", test.output, dst)
+			}
+		})
+	}
+}
+
+func TestParseMountPoints(t *testing.T) {
+	volumeMatch := regexp.MustCompile(`^\\\\\?\\Volume\{[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\}\\$`)
+
+	// It's not a good idea to test functions based on GetVolumeNameForVolumeMountPoint by calling
+	// GetVolumeNameForVolumeMountPoint itself, but we have restricted test environment:
+	// cannot manage volumes and can only be sure that the mount point C:\ exists
+	sysVolume, err := GetVolumeNameForVolumeMountPoint("C:")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// We don't know a valid volume GUID path for c:\, but we'll at least check its format
+	if !volumeMatch.MatchString(sysVolume) {
+		t.Fatalf("invalid volume GUID path: %s", sysVolume)
+	}
+	// Changing the case and removing trailing backslash allows tests
+	// the equality of different ways of writing a volume name
+	sysVolumeMutated := strings.ToUpper(sysVolume[:len(sysVolume)-1])
+	sysVolumeMatch := strings.ToLower(sysVolume)
+
+	type check struct {
+		volume string
+		result bool
+	}
+	setTests := []struct {
+		input  options.Options
+		output []string
+		checks []check
+		errors []string
+	}{
+		{
+			options.Options{
+				"vss.exclude-volumes": `c:;c:\;` + sysVolume + `;` + sysVolumeMutated,
+			},
+			[]string{
+				sysVolumeMatch,
+			},
+			[]check{
+				{`c:\`, false},
+				{`c:`, false},
+				{sysVolume, false},
+				{sysVolumeMutated, false},
+			},
+			[]string{},
+		},
+		{
+			options.Options{
+				"vss.exclude-volumes": `z:\nonexistent;c:;c:\windows\;\\?\Volume{39b9cac2-bcdb-4d51-97c8-0d0677d607fb}\`,
+			},
+			[]string{
+				sysVolumeMatch,
+			},
+			[]check{
+				{`c:\windows\`, true},
+				{`\\?\Volume{39b9cac2-bcdb-4d51-97c8-0d0677d607fb}\`, true},
+				{`c:`, false},
+				{``, true},
+			},
+			[]string{
+				`failed to parse vss\.exclude-volumes \[z:\\nonexistent\]:.*`,
+				`failed to parse vss\.exclude-volumes \[c:\\windows\\\]:.*`,
+				`failed to parse vss\.exclude-volumes \[\\\\\?\\Volume\{39b9cac2-bcdb-4d51-97c8-0d0677d607fb\}\\\]:.*`,
+				`failed to get volume from mount point \[c:\\windows\\\]:.*`,
+				`failed to get volume from mount point \[\\\\\?\\Volume\{39b9cac2-bcdb-4d51-97c8-0d0677d607fb\}\\\]:.*`,
+				`failed to get volume from mount point \[\]:.*`,
+			},
+		},
+	}
+
+	for i, test := range setTests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			cfg, err := ParseVSSConfig(test.input)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var log []string
+			errorHandler := func(item string, err error) {
+				log = append(log, strings.TrimSpace(err.Error()))
+			}
+			messageHandler := func(msg string, args ...interface{}) {
+				t.Fatalf("unexpected message (%s)", fmt.Sprintf(msg, args))
+			}
+
+			dst := NewLocalVss(errorHandler, messageHandler, cfg)
+
+			if !matchMap(test.output, dst.excludeVolumes) {
+				t.Fatalf("wrong result, want:\n  %#v\ngot:\n  %#v",
+					test.output, dst.excludeVolumes)
+			}
+
+			for _, c := range test.checks {
+				if dst.isMountPointIncluded(c.volume) != c.result {
+					t.Fatalf(`wrong check: isMountPointIncluded("%s") != %v`, c.volume, c.result)
+				}
+			}
+
+			if !matchStrings(test.errors, log) {
+				t.Fatalf("wrong log, want:\n  %#v\ngot:\n  %#v", test.errors, log)
+			}
+		})
+	}
+}
+
+func TestParseProvider(t *testing.T) {
+	msProvider := ole.NewGUID("{b5946137-7b9f-4925-af80-51abd60b20d5}")
+	setTests := []struct {
+		provider string
+		id       *ole.GUID
+		result   string
+	}{
+		{
+			"",
+			ole.IID_NULL,
+			"",
+		},
+		{
+			"mS",
+			msProvider,
+			"",
+		},
+		{
+			"{B5946137-7b9f-4925-Af80-51abD60b20d5}",
+			msProvider,
+			"",
+		},
+		{
+			"Microsoft Software Shadow Copy provider 1.0",
+			msProvider,
+			"",
+		},
+		{
+			"{04560982-3d7d-4bbc-84f7-0712f833a28f}",
+			nil,
+			`invalid VSS provider "{04560982-3d7d-4bbc-84f7-0712f833a28f}"`,
+		},
+		{
+			"non-existent provider",
+			nil,
+			`invalid VSS provider "non-existent provider"`,
+		},
+	}
+
+	_ = ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED)
+
+	for i, test := range setTests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			id, err := getProviderID(test.provider)
+
+			if err != nil && id != nil {
+				t.Fatalf("err!=nil but id=%v", id)
+			}
+
+			if test.result != "" || err != nil {
+				var result string
+				if err != nil {
+					result = err.Error()
+				}
+				if test.result != result || test.result == "" {
+					t.Fatalf("wrong result, want:\n  %#v\ngot:\n  %#v", test.result, result)
+				}
+			} else if !ole.IsEqualGUID(id, test.id) {
+				t.Fatalf("wrong id, want:\n  %s\ngot:\n  %s", test.id.String(), id.String())
+			}
+		})
+	}
+}

internal/fs/stat_test.go

@@ -5,11 +5,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 func TestExtendedStat(t *testing.T) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 	filename := filepath.Join(tempdir, "file")
 	err := os.WriteFile(filename, []byte("foobar"), 0640)
 	if err != nil {

internal/fs/vss.go

@@ -4,6 +4,8 @@
 package fs
 
 import (
+	"time"
+
 	"github.com/restic/restic/internal/errors"
 )
 
@@ -31,10 +33,16 @@ func HasSufficientPrivilegesForVSS() error {
 	return errors.New("VSS snapshots are only supported on windows")
 }
 
+// GetVolumeNameForVolumeMountPoint add trailing backslash to input parameter
+// and calls the equivalent windows api.
+func GetVolumeNameForVolumeMountPoint(mountPoint string) (string, error) {
+	return mountPoint, nil
+}
+
 // NewVssSnapshot creates a new vss snapshot. If creating the snapshots doesn't
 // finish within the timeout an error is returned.
-func NewVssSnapshot(
-	_ string, _ uint, _ ErrorHandler) (VssSnapshot, error) {
+func NewVssSnapshot(_ string,
+	_ string, _ time.Duration, _ VolumeFilter, _ ErrorHandler) (VssSnapshot, error) {
 	return VssSnapshot{}, errors.New("VSS snapshots are only supported on windows")
 }
 

internal/fs/vss_windows.go

@@ -5,10 +5,12 @@ package fs
 
 import (
 	"fmt"
+	"math"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"syscall"
+	"time"
 	"unsafe"
 
 	ole "github.com/go-ole/go-ole"
@@ -20,8 +22,10 @@ import (
 type HRESULT uint
 
 // HRESULT constant values necessary for using VSS api.
+//nolint:golint
 const (
 	S_OK                                            HRESULT = 0x00000000
+	S_FALSE                                         HRESULT = 0x00000001
 	E_ACCESSDENIED                                  HRESULT = 0x80070005
 	E_OUTOFMEMORY                                   HRESULT = 0x8007000E
 	E_INVALIDARG                                    HRESULT = 0x80070057
@@ -190,7 +194,7 @@ func (e *vssError) Error() string {
 	return fmt.Sprintf("VSS error: %s: %s (%#x)", e.text, e.hresult.Str(), e.hresult)
 }
 
-// VssError encapsulates errors returned from calling VSS api.
+// vssTextError encapsulates errors returned from calling VSS api.
 type vssTextError struct {
 	text string
 }
@@ -255,6 +259,7 @@ type IVssBackupComponents struct {
 }
 
 // IVssBackupComponentsVTable is the vtable for IVssBackupComponents.
+// nolint:structcheck
 type IVssBackupComponentsVTable struct {
 	ole.IUnknownVtbl
 	getWriterComponentsCount      uintptr
@@ -364,7 +369,7 @@ func (vss *IVssBackupComponents) convertToVSSAsync(
 }
 
 // IsVolumeSupported calls the equivalent VSS api.
-func (vss *IVssBackupComponents) IsVolumeSupported(volumeName string) (bool, error) {
+func (vss *IVssBackupComponents) IsVolumeSupported(providerID *ole.GUID, volumeName string) (bool, error) {
 	volumeNamePointer, err := syscall.UTF16PtrFromString(volumeName)
 	if err != nil {
 		panic(err)
@@ -374,7 +379,7 @@ func (vss *IVssBackupComponents) IsVolumeSupported(volumeName string) (bool, err
 	var result uintptr
 
 	if runtime.GOARCH == "386" {
-		id := (*[4]uintptr)(unsafe.Pointer(ole.IID_NULL))
+		id := (*[4]uintptr)(unsafe.Pointer(providerID))
 
 		result, _, _ = syscall.Syscall9(vss.getVTable().isVolumeSupported, 7,
 			uintptr(unsafe.Pointer(vss)), id[0], id[1], id[2], id[3],
@@ -382,7 +387,7 @@ func (vss *IVssBackupComponents) IsVolumeSupported(volumeName string) (bool, err
 			0)
 	} else {
 		result, _, _ = syscall.Syscall6(vss.getVTable().isVolumeSupported, 4,
-			uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(ole.IID_NULL)),
+			uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(providerID)),
 			uintptr(unsafe.Pointer(volumeNamePointer)), uintptr(unsafe.Pointer(&isSupportedRaw)), 0,
 			0)
 	}
@@ -408,24 +413,24 @@ func (vss *IVssBackupComponents) StartSnapshotSet() (ole.GUID, error) {
 }
 
 // AddToSnapshotSet calls the equivalent VSS api.
-func (vss *IVssBackupComponents) AddToSnapshotSet(volumeName string, idSnapshot *ole.GUID) error {
+func (vss *IVssBackupComponents) AddToSnapshotSet(volumeName string, providerID *ole.GUID, idSnapshot *ole.GUID) error {
 	volumeNamePointer, err := syscall.UTF16PtrFromString(volumeName)
 	if err != nil {
 		panic(err)
 	}
 
-	var result uintptr = 0
+	var result uintptr
 
 	if runtime.GOARCH == "386" {
-		id := (*[4]uintptr)(unsafe.Pointer(ole.IID_NULL))
+		id := (*[4]uintptr)(unsafe.Pointer(providerID))
 
 		result, _, _ = syscall.Syscall9(vss.getVTable().addToSnapshotSet, 7,
-			uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(volumeNamePointer)), id[0], id[1],
-			id[2], id[3], uintptr(unsafe.Pointer(idSnapshot)), 0, 0)
+			uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(volumeNamePointer)),
+			id[0], id[1], id[2], id[3], uintptr(unsafe.Pointer(idSnapshot)), 0, 0)
 	} else {
 		result, _, _ = syscall.Syscall6(vss.getVTable().addToSnapshotSet, 4,
 			uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(volumeNamePointer)),
-			uintptr(unsafe.Pointer(ole.IID_NULL)), uintptr(unsafe.Pointer(idSnapshot)), 0, 0)
+			uintptr(unsafe.Pointer(providerID)), uintptr(unsafe.Pointer(idSnapshot)), 0, 0)
 	}
 
 	return newVssErrorIfResultNotOK("AddToSnapshotSet() failed", HRESULT(result))
@@ -478,9 +483,9 @@ func (vss *IVssBackupComponents) DoSnapshotSet() (*IVSSAsync, error) {
 
 // DeleteSnapshots calls the equivalent VSS api.
 func (vss *IVssBackupComponents) DeleteSnapshots(snapshotID ole.GUID) (int32, ole.GUID, error) {
-	var deletedSnapshots int32 = 0
+	var deletedSnapshots int32
 	var nondeletedSnapshotID ole.GUID
-	var result uintptr = 0
+	var result uintptr
 
 	if runtime.GOARCH == "386" {
 		id := (*[4]uintptr)(unsafe.Pointer(&snapshotID))
@@ -504,7 +509,7 @@ func (vss *IVssBackupComponents) DeleteSnapshots(snapshotID ole.GUID) (int32, ol
 // GetSnapshotProperties calls the equivalent VSS api.
 func (vss *IVssBackupComponents) GetSnapshotProperties(snapshotID ole.GUID,
 	properties *VssSnapshotProperties) error {
-	var result uintptr = 0
+	var result uintptr
 
 	if runtime.GOARCH == "386" {
 		id := (*[4]uintptr)(unsafe.Pointer(&snapshotID))
@@ -527,8 +532,8 @@ func vssFreeSnapshotProperties(properties *VssSnapshotProperties) error {
 	if err != nil {
 		return err
 	}
-
-	proc.Call(uintptr(unsafe.Pointer(properties)))
+	// this function always succeeds and returns no value
+	_, _, _ = proc.Call(uintptr(unsafe.Pointer(properties)))
 	return nil
 }
 
@@ -543,6 +548,7 @@ func (vss *IVssBackupComponents) BackupComplete() (*IVSSAsync, error) {
 }
 
 // VssSnapshotProperties defines the properties of a VSS snapshot as part of the VSS api.
+// nolint:structcheck
 type VssSnapshotProperties struct {
 	snapshotID           ole.GUID
 	snapshotSetID        ole.GUID
@@ -559,6 +565,24 @@ type VssSnapshotProperties struct {
 	status               uint
 }
 
+// VssProviderProperties defines the properties of a VSS provider as part of the VSS api.
+// nolint:structcheck
+type VssProviderProperties struct {
+	providerID        ole.GUID
+	providerName      *uint16
+	providerType      uint32
+	providerVersion   *uint16
+	providerVersionID ole.GUID
+	classID           ole.GUID
+}
+
+func vssFreeProviderProperties(p *VssProviderProperties) {
+	ole.CoTaskMemFree(uintptr(unsafe.Pointer(p.providerName)))
+	p.providerName = nil
+	ole.CoTaskMemFree(uintptr(unsafe.Pointer(p.providerVersion)))
+	p.providerVersion = nil
+}
+
 // GetSnapshotDeviceObject returns root path to access the snapshot files
 // and folders.
 func (p *VssSnapshotProperties) GetSnapshotDeviceObject() string {
@@ -617,8 +641,13 @@ func (vssAsync *IVSSAsync) QueryStatus() (HRESULT, uint32) {
 
 // WaitUntilAsyncFinished waits until either the async call is finished or
 // the given timeout is reached.
-func (vssAsync *IVSSAsync) WaitUntilAsyncFinished(millis uint32) error {
-	hresult := vssAsync.Wait(millis)
+func (vssAsync *IVSSAsync) WaitUntilAsyncFinished(timeout time.Duration) error {
+	const maxTimeout = math.MaxInt32 * time.Millisecond
+	if timeout > maxTimeout {
+		timeout = maxTimeout
+	}
+
+	hresult := vssAsync.Wait(uint32(timeout.Milliseconds()))
 	err := newVssErrorIfResultNotOK("Wait() failed", hresult)
 	if err != nil {
 		vssAsync.Cancel()
@@ -651,6 +680,75 @@ func (vssAsync *IVSSAsync) WaitUntilAsyncFinished(millis uint32) error {
 	return nil
 }
 
+// UIID_IVSS_ADMIN defines the GUID of IVSSAdmin.
+var (
+	UIID_IVSS_ADMIN       = ole.NewGUID("{77ED5996-2F63-11d3-8A39-00C04F72D8E3}")
+	CLSID_VSS_COORDINATOR = ole.NewGUID("{E579AB5F-1CC4-44b4-BED9-DE0991FF0623}")
+)
+
+// IVSSAdmin VSS api interface.
+type IVSSAdmin struct {
+	ole.IUnknown
+}
+
+// IVSSAdminVTable is the vtable for IVSSAdmin.
+// nolint:structcheck
+type IVSSAdminVTable struct {
+	ole.IUnknownVtbl
+	registerProvider            uintptr
+	unregisterProvider          uintptr
+	queryProviders              uintptr
+	abortAllSnapshotsInProgress uintptr
+}
+
+// getVTable returns the vtable for IVSSAdmin.
+func (vssAdmin *IVSSAdmin) getVTable() *IVSSAdminVTable {
+	return (*IVSSAdminVTable)(unsafe.Pointer(vssAdmin.RawVTable))
+}
+
+// QueryProviders calls the equivalent VSS api.
+func (vssAdmin *IVSSAdmin) QueryProviders() (*IVssEnumObject, error) {
+	var enum *IVssEnumObject
+
+	result, _, _ := syscall.Syscall(vssAdmin.getVTable().queryProviders, 2,
+		uintptr(unsafe.Pointer(vssAdmin)), uintptr(unsafe.Pointer(&enum)), 0)
+
+	return enum, newVssErrorIfResultNotOK("QueryProviders() failed", HRESULT(result))
+}
+
+// IVssEnumObject VSS api interface.
+type IVssEnumObject struct {
+	ole.IUnknown
+}
+
+// IVssEnumObjectVTable is the vtable for IVssEnumObject.
+// nolint:structcheck
+type IVssEnumObjectVTable struct {
+	ole.IUnknownVtbl
+	next  uintptr
+	skip  uintptr
+	reset uintptr
+	clone uintptr
+}
+
+// getVTable returns the vtable for IVssEnumObject.
+func (vssEnum *IVssEnumObject) getVTable() *IVssEnumObjectVTable {
+	return (*IVssEnumObjectVTable)(unsafe.Pointer(vssEnum.RawVTable))
+}
+
+// Next calls the equivalent VSS api.
+func (vssEnum *IVssEnumObject) Next(count uint, props unsafe.Pointer) (uint, error) {
+	var fetched uint32
+	result, _, _ := syscall.Syscall6(vssEnum.getVTable().next, 4,
+		uintptr(unsafe.Pointer(vssEnum)), uintptr(count), uintptr(props),
+		uintptr(unsafe.Pointer(&fetched)), 0, 0)
+	if HRESULT(result) == S_FALSE {
+		return uint(fetched), nil
+	}
+
+	return uint(fetched), newVssErrorIfResultNotOK("Next() failed", HRESULT(result))
+}
+
 // MountPoint wraps all information of a snapshot of a mountpoint on a volume.
 type MountPoint struct {
 	isSnapshotted        bool
@@ -677,7 +775,7 @@ type VssSnapshot struct {
 	snapshotProperties   VssSnapshotProperties
 	snapshotDeviceObject string
 	mountPointInfo       map[string]MountPoint
-	timeoutInMillis      uint32
+	timeout              time.Duration
 }
 
 // GetSnapshotDeviceObject returns root path to access the snapshot files
@@ -694,7 +792,12 @@ func initializeVssCOMInterface() (*ole.IUnknown, error) {
 	}
 
 	// ensure COM is initialized before use
-	ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED)
+	if err = ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED); err != nil {
+		// CoInitializeEx returns S_FALSE if COM is already initialized
+		if oleErr, ok := err.(*ole.OleError); !ok || HRESULT(oleErr.Code()) != S_FALSE {
+			return nil, err
+		}
+	}
 
 	var oleIUnknown *ole.IUnknown
 	result, _, _ := vssInstance.Call(uintptr(unsafe.Pointer(&oleIUnknown)))
@@ -727,12 +830,34 @@ func HasSufficientPrivilegesForVSS() error {
 	return err
 }
 
+// GetVolumeNameForVolumeMountPoint add trailing backslash to input parameter
+// and calls the equivalent windows api.
+func GetVolumeNameForVolumeMountPoint(mountPoint string) (string, error) {
+	if mountPoint != "" && mountPoint[len(mountPoint)-1] != filepath.Separator {
+		mountPoint += string(filepath.Separator)
+	}
+
+	mountPointPointer, err := syscall.UTF16PtrFromString(mountPoint)
+	if err != nil {
+		return mountPoint, err
+	}
+
+	// A reasonable size for the buffer to accommodate the largest possible
+	// volume GUID path is 50 characters.
+	volumeNameBuffer := make([]uint16, 50)
+	if err := windows.GetVolumeNameForVolumeMountPoint(
+		mountPointPointer, &volumeNameBuffer[0], 50); err != nil {
+		return mountPoint, err
+	}
+
+	return syscall.UTF16ToString(volumeNameBuffer), nil
+}
+
 // NewVssSnapshot creates a new vss snapshot. If creating the snapshots doesn't
 // finish within the timeout an error is returned.
-func NewVssSnapshot(
-	volume string, timeoutInSeconds uint, msgError ErrorHandler) (VssSnapshot, error) {
+func NewVssSnapshot(provider string,
+	volume string, timeout time.Duration, filter VolumeFilter, msgError ErrorHandler) (VssSnapshot, error) {
 	is64Bit, err := isRunningOn64BitWindows()
-
 	if err != nil {
 		return VssSnapshot{}, newVssTextError(fmt.Sprintf(
 			"Failed to detect windows architecture: %s", err.Error()))
@@ -744,7 +869,7 @@ func NewVssSnapshot(
 			runtime.GOARCH))
 	}
 
-	timeoutInMillis := uint32(timeoutInSeconds * 1000)
+	deadline := time.Now().Add(timeout)
 
 	oleIUnknown, err := initializeVssCOMInterface()
 	if oleIUnknown != nil {
@@ -778,6 +903,12 @@ func NewVssSnapshot(
 
 	iVssBackupComponents := (*IVssBackupComponents)(unsafe.Pointer(comInterface))
 
+	providerID, err := getProviderID(provider)
+	if err != nil {
+		iVssBackupComponents.Release()
+		return VssSnapshot{}, err
+	}
+
 	if err := iVssBackupComponents.InitializeForBackup(); err != nil {
 		iVssBackupComponents.Release()
 		return VssSnapshot{}, err
@@ -796,13 +927,13 @@ func NewVssSnapshot(
 	}
 
 	err = callAsyncFunctionAndWait(iVssBackupComponents.GatherWriterMetadata,
-		"GatherWriterMetadata", timeoutInMillis)
+		"GatherWriterMetadata", deadline)
 	if err != nil {
 		iVssBackupComponents.Release()
 		return VssSnapshot{}, err
 	}
 
-	if isSupported, err := iVssBackupComponents.IsVolumeSupported(volume); err != nil {
+	if isSupported, err := iVssBackupComponents.IsVolumeSupported(providerID, volume); err != nil {
 		iVssBackupComponents.Release()
 		return VssSnapshot{}, err
 	} else if !isSupported {
@@ -817,44 +948,53 @@ func NewVssSnapshot(
 		return VssSnapshot{}, err
 	}
 
-	if err := iVssBackupComponents.AddToSnapshotSet(volume, &snapshotSetID); err != nil {
+	if err := iVssBackupComponents.AddToSnapshotSet(volume, providerID, &snapshotSetID); err != nil {
 		iVssBackupComponents.Release()
 		return VssSnapshot{}, err
 	}
 
-	mountPoints, err := enumerateMountedFolders(volume)
-	if err != nil {
-		iVssBackupComponents.Release()
-		return VssSnapshot{}, newVssTextError(fmt.Sprintf(
-			"failed to enumerate mount points for volume %s: %s", volume, err))
-	}
-
 	mountPointInfo := make(map[string]MountPoint)
 
-	for _, mountPoint := range mountPoints {
-		// ensure every mountpoint is available even without a valid
-		// snapshot because we need to consider this when backing up files
-		mountPointInfo[mountPoint] = MountPoint{isSnapshotted: false}
-
-		if isSupported, err := iVssBackupComponents.IsVolumeSupported(mountPoint); err != nil {
-			continue
-		} else if !isSupported {
-			continue
-		}
-
-		var mountPointSnapshotSetID ole.GUID
-		err := iVssBackupComponents.AddToSnapshotSet(mountPoint, &mountPointSnapshotSetID)
+	// if filter==nil just don't process mount points for this volume at all
+	if filter != nil {
+		mountPoints, err := enumerateMountedFolders(volume)
 		if err != nil {
 			iVssBackupComponents.Release()
-			return VssSnapshot{}, err
+
+			return VssSnapshot{}, newVssTextError(fmt.Sprintf(
+				"failed to enumerate mount points for volume %s: %s", volume, err))
 		}
 
-		mountPointInfo[mountPoint] = MountPoint{isSnapshotted: true,
-			snapshotSetID: mountPointSnapshotSetID}
+		for _, mountPoint := range mountPoints {
+			// ensure every mountpoint is available even without a valid
+			// snapshot because we need to consider this when backing up files
+			mountPointInfo[mountPoint] = MountPoint{isSnapshotted: false}
+
+			if !filter(mountPoint) {
+				continue
+			} else if isSupported, err := iVssBackupComponents.IsVolumeSupported(providerID, mountPoint); err != nil {
+				continue
+			} else if !isSupported {
+				continue
+			}
+
+			var mountPointSnapshotSetID ole.GUID
+			err := iVssBackupComponents.AddToSnapshotSet(mountPoint, providerID, &mountPointSnapshotSetID)
+			if err != nil {
+				iVssBackupComponents.Release()
+
+				return VssSnapshot{}, err
+			}
+
+			mountPointInfo[mountPoint] = MountPoint{
+				isSnapshotted: true,
+				snapshotSetID: mountPointSnapshotSetID,
+			}
+		}
 	}
 
 	err = callAsyncFunctionAndWait(iVssBackupComponents.PrepareForBackup, "PrepareForBackup",
-		timeoutInMillis)
+		deadline)
 	if err != nil {
 		// After calling PrepareForBackup one needs to call AbortBackup() before releasing the VSS
 		// instance for proper cleanup.
@@ -865,9 +1005,9 @@ func NewVssSnapshot(
 	}
 
 	err = callAsyncFunctionAndWait(iVssBackupComponents.DoSnapshotSet, "DoSnapshotSet",
-		timeoutInMillis)
+		deadline)
 	if err != nil {
-		iVssBackupComponents.AbortBackup()
+		_ = iVssBackupComponents.AbortBackup()
 		iVssBackupComponents.Release()
 		return VssSnapshot{}, err
 	}
@@ -875,13 +1015,12 @@ func NewVssSnapshot(
 	var snapshotProperties VssSnapshotProperties
 	err = iVssBackupComponents.GetSnapshotProperties(snapshotSetID, &snapshotProperties)
 	if err != nil {
-		iVssBackupComponents.AbortBackup()
+		_ = iVssBackupComponents.AbortBackup()
 		iVssBackupComponents.Release()
 		return VssSnapshot{}, err
 	}
 
 	for mountPoint, info := range mountPointInfo {
-
 		if !info.isSnapshotted {
 			continue
 		}
@@ -900,8 +1039,10 @@ func NewVssSnapshot(
 		mountPointInfo[mountPoint] = info
 	}
 
-	return VssSnapshot{iVssBackupComponents, snapshotSetID, snapshotProperties,
-		snapshotProperties.GetSnapshotDeviceObject(), mountPointInfo, timeoutInMillis}, nil
+	return VssSnapshot{
+		iVssBackupComponents, snapshotSetID, snapshotProperties,
+		snapshotProperties.GetSnapshotDeviceObject(), mountPointInfo, time.Until(deadline),
+	}, nil
 }
 
 // Delete deletes the created snapshot.
@@ -922,15 +1063,17 @@ func (p *VssSnapshot) Delete() error {
 	if p.iVssBackupComponents != nil {
 		defer p.iVssBackupComponents.Release()
 
+		deadline := time.Now().Add(p.timeout)
+
 		err = callAsyncFunctionAndWait(p.iVssBackupComponents.BackupComplete, "BackupComplete",
-			p.timeoutInMillis)
+			deadline)
 		if err != nil {
 			return err
 		}
 
 		if _, _, e := p.iVssBackupComponents.DeleteSnapshots(p.snapshotID); e != nil {
 			err = newVssTextError(fmt.Sprintf("Failed to delete snapshot: %s", e.Error()))
-			p.iVssBackupComponents.AbortBackup()
+			_ = p.iVssBackupComponents.AbortBackup()
 			if err != nil {
 				return err
 			}
@@ -940,12 +1083,61 @@ func (p *VssSnapshot) Delete() error {
 	return nil
 }
 
+func getProviderID(provider string) (*ole.GUID, error) {
+	providerLower := strings.ToLower(provider)
+	switch providerLower {
+	case "":
+		return ole.IID_NULL, nil
+	case "ms":
+		return ole.NewGUID("{b5946137-7b9f-4925-af80-51abd60b20d5}"), nil
+	}
+
+	comInterface, err := ole.CreateInstance(CLSID_VSS_COORDINATOR, UIID_IVSS_ADMIN)
+	if err != nil {
+		return nil, err
+	}
+	defer comInterface.Release()
+
+	vssAdmin := (*IVSSAdmin)(unsafe.Pointer(comInterface))
+
+	enum, err := vssAdmin.QueryProviders()
+	if err != nil {
+		return nil, err
+	}
+	defer enum.Release()
+
+	id := ole.NewGUID(provider)
+
+	var props struct {
+		objectType uint32
+		provider   VssProviderProperties
+	}
+	for {
+		count, err := enum.Next(1, unsafe.Pointer(&props))
+		if err != nil {
+			return nil, err
+		}
+
+		if count < 1 {
+			return nil, errors.Errorf(`invalid VSS provider "%s"`, provider)
+		}
+
+		name := ole.UTF16PtrToString(props.provider.providerName)
+		vssFreeProviderProperties(&props.provider)
+
+		if id != nil && *id == props.provider.providerID ||
+			id == nil && providerLower == strings.ToLower(name) {
+			return &props.provider.providerID, nil
+		}
+	}
+}
+
 // asyncCallFunc is the callback type for callAsyncFunctionAndWait.
 type asyncCallFunc func() (*IVSSAsync, error)
 
 // callAsyncFunctionAndWait calls an async functions and waits for it to either
 // finish or timeout.
-func callAsyncFunctionAndWait(function asyncCallFunc, name string, timeoutInMillis uint32) error {
+func callAsyncFunctionAndWait(function asyncCallFunc, name string, deadline time.Time) error {
 	iVssAsync, err := function()
 	if err != nil {
 		return err
@@ -955,7 +1147,12 @@ func callAsyncFunctionAndWait(function asyncCallFunc, name string, timeoutInMill
 		return newVssTextError(fmt.Sprintf("%s() returned nil", name))
 	}
 
-	err = iVssAsync.WaitUntilAsyncFinished(timeoutInMillis)
+	timeout := time.Until(deadline)
+	if timeout <= 0 {
+		return newVssTextError(fmt.Sprintf("%s() deadline exceeded", name))
+	}
+
+	err = iVssAsync.WaitUntilAsyncFinished(timeout)
 	iVssAsync.Release()
 	return err
 }
@@ -1036,6 +1233,7 @@ func enumerateMountedFolders(volume string) ([]string, error) {
 		return mountedFolders, nil
 	}
 
+	// nolint:errcheck
 	defer windows.FindVolumeMountPointClose(handle)
 
 	volumeMountPoint := syscall.UTF16ToString(volumeMountPointBuffer)

internal/index/index.go

@@ -3,12 +3,15 @@ package index
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
+	"math"
 	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/restic"
 
 	"github.com/restic/restic/internal/debug"
@@ -67,11 +70,9 @@ func (idx *Index) addToPacks(id restic.ID) int {
 	return len(idx.packs) - 1
 }
 
-const maxuint32 = 1<<32 - 1
-
 func (idx *Index) store(packIndex int, blob restic.Blob) {
 	// assert that offset and length fit into uint32!
-	if blob.Offset > maxuint32 || blob.Length > maxuint32 || blob.UncompressedLength > maxuint32 {
+	if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 {
 		panic("offset or length does not fit in uint32. You have packs > 4GB!")
 	}
 
@@ -217,7 +218,7 @@ func (idx *Index) AddToSupersedes(ids ...restic.ID) error {
 
 // Each passes all blobs known to the index to the callback fn. This blocks any
 // modification of the index.
-func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) {
+func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) error {
 	idx.m.Lock()
 	defer idx.m.Unlock()
 
@@ -231,6 +232,7 @@ func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) {
 			return true
 		})
 	}
+	return ctx.Err()
 }
 
 type EachByPackResult struct {
@@ -515,8 +517,13 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro
 		debug.Log("Error %v", err)
 
 		if isErrOldIndex(err) {
+			if feature.Flag.Enabled(feature.DeprecateLegacyIndex) {
+				return nil, false, fmt.Errorf("index seems to use the legacy format. update it using `restic repair index`")
+			}
+
 			debug.Log("index is probably old format, trying that")
 			idx, err = decodeOldIndex(buf)
+			idx.ids = append(idx.ids, id)
 			return idx, err == nil, err
 		}
 

internal/index/index_parallel_test.go

@@ -15,11 +15,9 @@ import (
 var repoFixture = filepath.Join("..", "repository", "testdata", "test-repo.tar.gz")
 
 func TestRepositoryForAllIndexes(t *testing.T) {
-	repodir, cleanup := rtest.Env(t, repoFixture)
+	repo, cleanup := repository.TestFromFixture(t, repoFixture)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	expectedIndexIDs := restic.NewIDSet()
 	rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
 		expectedIndexIDs.Insert(id)

internal/index/index_test.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -338,7 +339,7 @@ func TestIndexUnserialize(t *testing.T) {
 
 		rtest.Equals(t, oldIdx, idx.Supersedes())
 
-		blobs := listPack(idx, exampleLookupTest.packID)
+		blobs := listPack(t, idx, exampleLookupTest.packID)
 		if len(blobs) != len(exampleLookupTest.blobs) {
 			t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs))
 		}
@@ -355,12 +356,12 @@ func TestIndexUnserialize(t *testing.T) {
 	}
 }
 
-func listPack(idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
-	idx.Each(context.TODO(), func(pb restic.PackedBlob) {
+func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
+	rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) {
 		if pb.PackID.Equal(id) {
 			pbs = append(pbs, pb)
 		}
-	})
+	}))
 	return pbs
 }
 
@@ -427,6 +428,8 @@ func BenchmarkEncodeIndex(b *testing.B) {
 }
 
 func TestIndexUnserializeOld(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateLegacyIndex, false)()
+
 	idx, oldFormat, err := index.DecodeIndex(docOldExample, restic.NewRandomID())
 	rtest.OK(t, err)
 	rtest.Assert(t, oldFormat, "old index format recognized as new format")

internal/index/master_index.go

@@ -223,13 +223,16 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index {
 
 // Each runs fn on all blobs known to the index. When the context is cancelled,
 // the index iteration return immediately. This blocks any modification of the index.
-func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) {
+func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) error {
 	mi.idxMutex.RLock()
 	defer mi.idxMutex.RUnlock()
 
 	for _, idx := range mi.idx {
-		idx.Each(ctx, fn)
+		if err := idx.Each(ctx, fn); err != nil {
+			return err
+		}
 	}
+	return nil
 }
 
 // MergeFinalIndexes merges all final indexes together.
@@ -320,6 +323,9 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, exclude
 					newIndex = NewIndex()
 				}
 			}
+			if wgCtx.Err() != nil {
+				return wgCtx.Err()
+			}
 		}
 
 		err := newIndex.AddToSupersedes(extraObsolete...)
@@ -426,10 +432,6 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
 		defer close(out)
 		// only resort a part of the index to keep the memory overhead bounded
 		for i := byte(0); i < 16; i++ {
-			if ctx.Err() != nil {
-				return
-			}
-
 			packBlob := make(map[restic.ID][]restic.Blob)
 			for pack := range packs {
 				if pack[0]&0xf == i {
@@ -439,11 +441,14 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
 			if len(packBlob) == 0 {
 				continue
 			}
-			mi.Each(ctx, func(pb restic.PackedBlob) {
+			err := mi.Each(ctx, func(pb restic.PackedBlob) {
 				if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i {
 					packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob)
 				}
 			})
+			if err != nil {
+				return
+			}
 
 			// pass on packs
 			for packID, pbs := range packBlob {

internal/index/master_index_test.go

@@ -166,9 +166,9 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
 	rtest.Equals(t, 1, idxCount)
 
 	blobCount := 0
-	mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
+	rtest.OK(t, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
 		blobCount++
-	})
+	}))
 	rtest.Equals(t, 2, blobCount)
 
 	blobs := mIdx.Lookup(bhInIdx1)
@@ -198,9 +198,9 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
 	rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
 
 	blobCount = 0
-	mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
+	rtest.OK(t, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
 		blobCount++
-	})
+	}))
 	rtest.Equals(t, 2, blobCount)
 }
 
@@ -319,9 +319,9 @@ func BenchmarkMasterIndexEach(b *testing.B) {
 
 	for i := 0; i < b.N; i++ {
 		entries := 0
-		mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
+		rtest.OK(b, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
 			entries++
-		})
+		}))
 	}
 }
 

internal/pack/pack.go

@@ -389,10 +389,10 @@ func CalculateHeaderSize(blobs []restic.Blob) int {
 // If onlyHdr is set to true, only the size of the header is returned
 // Note that this function only gives correct sizes, if there are no
 // duplicates in the index.
-func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.ID]int64 {
+func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) (map[restic.ID]int64, error) {
 	packSize := make(map[restic.ID]int64)
 
-	mi.Each(ctx, func(blob restic.PackedBlob) {
+	err := mi.Each(ctx, func(blob restic.PackedBlob) {
 		size, ok := packSize[blob.PackID]
 		if !ok {
 			size = headerSize
@@ -403,5 +403,5 @@ func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.I
 		packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob))
 	})
 
-	return packSize
+	return packSize, err
 }

internal/repository/key.go

@@ -43,11 +43,11 @@ type Key struct {
 	id restic.ID
 }
 
-// Params tracks the parameters used for the KDF. If not set, it will be
+// params tracks the parameters used for the KDF. If not set, it will be
 // calibrated on the first run of AddKey().
-var Params *crypto.Params
+var params *crypto.Params
 
-var (
+const (
 	// KDFTimeout specifies the maximum runtime for the KDF.
 	KDFTimeout = 500 * time.Millisecond
 
@@ -196,13 +196,13 @@ func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err erro
 // AddKey adds a new key to an already existing repository.
 func AddKey(ctx context.Context, s *Repository, password, username, hostname string, template *crypto.Key) (*Key, error) {
 	// make sure we have valid KDF parameters
-	if Params == nil {
+	if params == nil {
 		p, err := crypto.Calibrate(KDFTimeout, KDFMemory)
 		if err != nil {
 			return nil, errors.Wrap(err, "Calibrate")
 		}
 
-		Params = &p
+		params = &p
 		debug.Log("calibrated KDF parameters are %v", p)
 	}
 
@@ -213,9 +213,9 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
 		Hostname: hostname,
 
 		KDF: "scrypt",
-		N:   Params.N,
-		R:   Params.R,
-		P:   Params.P,
+		N:   params.N,
+		R:   params.R,
+		P:   params.P,
 	}
 
 	if newkey.Hostname == "" {
@@ -237,7 +237,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
 	}
 
 	// call KDF to derive user key
-	newkey.user, err = crypto.KDF(*Params, newkey.Salt, password)
+	newkey.user, err = crypto.KDF(*params, newkey.Salt, password)
 	if err != nil {
 		return nil, err
 	}

internal/repository/lock.go

@@ -0,0 +1,274 @@
+package repository
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
+)
+
+type lockContext struct {
+	lock      *restic.Lock
+	cancel    context.CancelFunc
+	refreshWG sync.WaitGroup
+}
+
+type locker struct {
+	retrySleepStart       time.Duration
+	retrySleepMax         time.Duration
+	refreshInterval       time.Duration
+	refreshabilityTimeout time.Duration
+}
+
+const defaultRefreshInterval = 5 * time.Minute
+
+var lockerInst = &locker{
+	retrySleepStart: 5 * time.Second,
+	retrySleepMax:   60 * time.Second,
+	refreshInterval: defaultRefreshInterval,
+	// consider a lock refresh failed a bit before the lock actually becomes stale
+	// the difference allows to compensate for a small time drift between clients.
+	refreshabilityTimeout: restic.StaleLockTimeout - defaultRefreshInterval*3/2,
+}
+
+func Lock(ctx context.Context, repo restic.Repository, exclusive bool, retryLock time.Duration, printRetry func(msg string), logger func(format string, args ...interface{})) (*Unlocker, context.Context, error) {
+	return lockerInst.Lock(ctx, repo, exclusive, retryLock, printRetry, logger)
+}
+
+// Lock wraps the ctx such that it is cancelled when the repository is unlocked
+// cancelling the original context also stops the lock refresh
+func (l *locker) Lock(ctx context.Context, repo restic.Repository, exclusive bool, retryLock time.Duration, printRetry func(msg string), logger func(format string, args ...interface{})) (*Unlocker, context.Context, error) {
+
+	lockFn := restic.NewLock
+	if exclusive {
+		lockFn = restic.NewExclusiveLock
+	}
+
+	var lock *restic.Lock
+	var err error
+
+	retrySleep := minDuration(l.retrySleepStart, retryLock)
+	retryMessagePrinted := false
+	retryTimeout := time.After(retryLock)
+
+retryLoop:
+	for {
+		lock, err = lockFn(ctx, repo)
+		if err != nil && restic.IsAlreadyLocked(err) {
+
+			if !retryMessagePrinted {
+				printRetry(fmt.Sprintf("repo already locked, waiting up to %s for the lock\n", retryLock))
+				retryMessagePrinted = true
+			}
+
+			debug.Log("repo already locked, retrying in %v", retrySleep)
+			retrySleepCh := time.After(retrySleep)
+
+			select {
+			case <-ctx.Done():
+				return nil, ctx, ctx.Err()
+			case <-retryTimeout:
+				debug.Log("repo already locked, timeout expired")
+				// Last lock attempt
+				lock, err = lockFn(ctx, repo)
+				break retryLoop
+			case <-retrySleepCh:
+				retrySleep = minDuration(retrySleep*2, l.retrySleepMax)
+			}
+		} else {
+			// anything else, either a successful lock or another error
+			break retryLoop
+		}
+	}
+	if restic.IsInvalidLock(err) {
+		return nil, ctx, errors.Fatalf("%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command", err)
+	}
+	if err != nil {
+		return nil, ctx, fmt.Errorf("unable to create lock in backend: %w", err)
+	}
+	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
+
+	ctx, cancel := context.WithCancel(ctx)
+	lockInfo := &lockContext{
+		lock:   lock,
+		cancel: cancel,
+	}
+	lockInfo.refreshWG.Add(2)
+	refreshChan := make(chan struct{})
+	forceRefreshChan := make(chan refreshLockRequest)
+
+	go l.refreshLocks(ctx, repo.Backend(), lockInfo, refreshChan, forceRefreshChan, logger)
+	go l.monitorLockRefresh(ctx, lockInfo, refreshChan, forceRefreshChan, logger)
+
+	return &Unlocker{lockInfo}, ctx, nil
+}
+
+func minDuration(a, b time.Duration) time.Duration {
+	if a <= b {
+		return a
+	}
+	return b
+}
+
+type refreshLockRequest struct {
+	result chan bool
+}
+
+func (l *locker) refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest, logger func(format string, args ...interface{})) {
+	debug.Log("start")
+	lock := lockInfo.lock
+	ticker := time.NewTicker(l.refreshInterval)
+	lastRefresh := lock.Time
+
+	defer func() {
+		ticker.Stop()
+		// ensure that the context was cancelled before removing the lock
+		lockInfo.cancel()
+
+		// remove the lock from the repo
+		debug.Log("unlocking repository with lock %v", lock)
+		if err := lock.Unlock(); err != nil {
+			debug.Log("error while unlocking: %v", err)
+			logger("error while unlocking: %v", err)
+		}
+
+		lockInfo.refreshWG.Done()
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			debug.Log("terminate")
+			return
+
+		case req := <-forceRefresh:
+			debug.Log("trying to refresh stale lock")
+			// keep on going if our current lock still exists
+			success := tryRefreshStaleLock(ctx, backend, lock, lockInfo.cancel, logger)
+			// inform refresh goroutine about forced refresh
+			select {
+			case <-ctx.Done():
+			case req.result <- success:
+			}
+
+			if success {
+				// update lock refresh time
+				lastRefresh = lock.Time
+			}
+
+		case <-ticker.C:
+			if time.Since(lastRefresh) > l.refreshabilityTimeout {
+				// the lock is too old, wait until the expiry monitor cancels the context
+				continue
+			}
+
+			debug.Log("refreshing locks")
+			err := lock.Refresh(context.TODO())
+			if err != nil {
+				logger("unable to refresh lock: %v\n", err)
+			} else {
+				lastRefresh = lock.Time
+				// inform monitor goroutine about successful refresh
+				select {
+				case <-ctx.Done():
+				case refreshed <- struct{}{}:
+				}
+			}
+		}
+	}
+}
+
+func (l *locker) monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-chan struct{}, forceRefresh chan<- refreshLockRequest, logger func(format string, args ...interface{})) {
+	// time.Now() might use a monotonic timer which is paused during standby
+	// convert to unix time to ensure we compare real time values
+	lastRefresh := time.Now().UnixNano()
+	pollDuration := 1 * time.Second
+	if l.refreshInterval < pollDuration {
+		// required for TestLockFailedRefresh
+		pollDuration = l.refreshInterval / 5
+	}
+	// timers are paused during standby, which is a problem as the refresh timeout
+	// _must_ expire if the host was too long in standby. Thus fall back to periodic checks
+	// https://github.com/golang/go/issues/35012
+	ticker := time.NewTicker(pollDuration)
+	defer func() {
+		ticker.Stop()
+		lockInfo.cancel()
+		lockInfo.refreshWG.Done()
+	}()
+
+	var refreshStaleLockResult chan bool
+
+	for {
+		select {
+		case <-ctx.Done():
+			debug.Log("terminate expiry monitoring")
+			return
+		case <-refreshed:
+			if refreshStaleLockResult != nil {
+				// ignore delayed refresh notifications while the stale lock is refreshed
+				continue
+			}
+			lastRefresh = time.Now().UnixNano()
+		case <-ticker.C:
+			if time.Now().UnixNano()-lastRefresh < l.refreshabilityTimeout.Nanoseconds() || refreshStaleLockResult != nil {
+				continue
+			}
+
+			debug.Log("trying to refreshStaleLock")
+			// keep on going if our current lock still exists
+			refreshReq := refreshLockRequest{
+				result: make(chan bool),
+			}
+			refreshStaleLockResult = refreshReq.result
+
+			// inform refresh goroutine about forced refresh
+			select {
+			case <-ctx.Done():
+			case forceRefresh <- refreshReq:
+			}
+		case success := <-refreshStaleLockResult:
+			if success {
+				lastRefresh = time.Now().UnixNano()
+				refreshStaleLockResult = nil
+				continue
+			}
+
+			logger("Fatal: failed to refresh lock in time\n")
+			return
+		}
+	}
+}
+
+func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc, logger func(format string, args ...interface{})) bool {
+	freeze := backend.AsBackend[backend.FreezeBackend](be)
+	if freeze != nil {
+		debug.Log("freezing backend")
+		freeze.Freeze()
+		defer freeze.Unfreeze()
+	}
+
+	err := lock.RefreshStaleLock(ctx)
+	if err != nil {
+		logger("failed to refresh stale lock: %v\n", err)
+		// cancel context while the backend is still frozen to prevent accidental modifications
+		cancel()
+		return false
+	}
+
+	return true
+}
+
+type Unlocker struct {
+	info *lockContext
+}
+
+func (l *Unlocker) Unlock() {
+	l.info.cancel()
+	l.info.refreshWG.Wait()
+}

internal/repository/lock_test.go

@@ -0,0 +1,304 @@
+package repository
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/backend/mem"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+type backendWrapper func(r backend.Backend) (backend.Backend, error)
+
+func openLockTestRepo(t *testing.T, wrapper backendWrapper) restic.Repository {
+	be := backend.Backend(mem.New())
+	// initialize repo
+	TestRepositoryWithBackend(t, be, 0, Options{})
+
+	// reopen repository to allow injecting a backend wrapper
+	if wrapper != nil {
+		var err error
+		be, err = wrapper(be)
+		rtest.OK(t, err)
+	}
+
+	return TestOpenBackend(t, be)
+}
+
+func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository, lockerInst *locker, retryLock time.Duration) (*Unlocker, context.Context) {
+	lock, wrappedCtx, err := lockerInst.Lock(ctx, repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {})
+	test.OK(t, err)
+	test.OK(t, wrappedCtx.Err())
+	if lock.info.lock.Stale() {
+		t.Fatal("lock returned stale lock")
+	}
+	return lock, wrappedCtx
+}
+
+func TestLock(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, nil)
+
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, lockerInst, 0)
+	lock.Unlock()
+	if wrappedCtx.Err() == nil {
+		t.Fatal("unlock did not cancel context")
+	}
+}
+
+func TestLockCancel(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, nil)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	lock, wrappedCtx := checkedLockRepo(ctx, t, repo, lockerInst, 0)
+	cancel()
+	if wrappedCtx.Err() == nil {
+		t.Fatal("canceled parent context did not cancel context")
+	}
+
+	// Unlock should not crash
+	lock.Unlock()
+}
+
+func TestLockConflict(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, nil)
+	repo2 := TestOpenBackend(t, repo.Backend())
+
+	lock, _, err := Lock(context.Background(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {})
+	test.OK(t, err)
+	defer lock.Unlock()
+	_, _, err = Lock(context.Background(), repo2, false, 0, func(msg string) {}, func(format string, args ...interface{}) {})
+	if err == nil {
+		t.Fatal("second lock should have failed")
+	}
+	test.Assert(t, restic.IsAlreadyLocked(err), "unexpected error %v", err)
+}
+
+type writeOnceBackend struct {
+	backend.Backend
+	written bool
+}
+
+func (b *writeOnceBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+	if b.written {
+		return fmt.Errorf("fail after first write")
+	}
+	b.written = true
+	return b.Backend.Save(ctx, h, rd)
+}
+
+func TestLockFailedRefresh(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
+		return &writeOnceBackend{Backend: r}, nil
+	})
+
+	// reduce locking intervals to be suitable for testing
+	li := &locker{
+		retrySleepStart:       lockerInst.retrySleepStart,
+		retrySleepMax:         lockerInst.retrySleepMax,
+		refreshInterval:       20 * time.Millisecond,
+		refreshabilityTimeout: 100 * time.Millisecond,
+	}
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, li, 0)
+
+	select {
+	case <-wrappedCtx.Done():
+		// expected lock refresh failure
+	case <-time.After(time.Second):
+		t.Fatal("failed lock refresh did not cause context cancellation")
+	}
+	// Unlock should not crash
+	lock.Unlock()
+}
+
+type loggingBackend struct {
+	backend.Backend
+	t *testing.T
+}
+
+func (b *loggingBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+	b.t.Logf("save %v @ %v", h, time.Now())
+	err := b.Backend.Save(ctx, h, rd)
+	b.t.Logf("save finished %v @ %v", h, time.Now())
+	return err
+}
+
+func TestLockSuccessfulRefresh(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
+		return &loggingBackend{
+			Backend: r,
+			t:       t,
+		}, nil
+	})
+
+	t.Logf("test for successful lock refresh %v", time.Now())
+	// reduce locking intervals to be suitable for testing
+	li := &locker{
+		retrySleepStart:       lockerInst.retrySleepStart,
+		retrySleepMax:         lockerInst.retrySleepMax,
+		refreshInterval:       60 * time.Millisecond,
+		refreshabilityTimeout: 500 * time.Millisecond,
+	}
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, li, 0)
+
+	select {
+	case <-wrappedCtx.Done():
+		// don't call t.Fatal to allow the lock to be properly cleaned up
+		t.Error("lock refresh failed", time.Now())
+
+		// Dump full stacktrace
+		buf := make([]byte, 1024*1024)
+		n := runtime.Stack(buf, true)
+		buf = buf[:n]
+		t.Log(string(buf))
+
+	case <-time.After(2 * li.refreshabilityTimeout):
+		// expected lock refresh to work
+	}
+	// Unlock should not crash
+	lock.Unlock()
+}
+
+type slowBackend struct {
+	backend.Backend
+	m     sync.Mutex
+	sleep time.Duration
+}
+
+func (b *slowBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
+	b.m.Lock()
+	sleep := b.sleep
+	b.m.Unlock()
+	time.Sleep(sleep)
+	return b.Backend.Save(ctx, h, rd)
+}
+
+func TestLockSuccessfulStaleRefresh(t *testing.T) {
+	t.Parallel()
+	var sb *slowBackend
+	repo := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) {
+		sb = &slowBackend{Backend: r}
+		return sb, nil
+	})
+
+	t.Logf("test for successful lock refresh %v", time.Now())
+	// reduce locking intervals to be suitable for testing
+	li := &locker{
+		retrySleepStart:       lockerInst.retrySleepStart,
+		retrySleepMax:         lockerInst.retrySleepMax,
+		refreshInterval:       10 * time.Millisecond,
+		refreshabilityTimeout: 50 * time.Millisecond,
+	}
+
+	lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, li, 0)
+	// delay lock refreshing long enough that the lock would expire
+	sb.m.Lock()
+	sb.sleep = li.refreshabilityTimeout + li.refreshInterval
+	sb.m.Unlock()
+
+	select {
+	case <-wrappedCtx.Done():
+		// don't call t.Fatal to allow the lock to be properly cleaned up
+		t.Error("lock refresh failed", time.Now())
+
+	case <-time.After(li.refreshabilityTimeout):
+	}
+	// reset slow backend
+	sb.m.Lock()
+	sb.sleep = 0
+	sb.m.Unlock()
+	debug.Log("normal lock period has expired")
+
+	select {
+	case <-wrappedCtx.Done():
+		// don't call t.Fatal to allow the lock to be properly cleaned up
+		t.Error("lock refresh failed", time.Now())
+
+	case <-time.After(3 * li.refreshabilityTimeout):
+		// expected lock refresh to work
+	}
+
+	// Unlock should not crash
+	lock.Unlock()
+}
+
+func TestLockWaitTimeout(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, nil)
+
+	elock, _, err := Lock(context.TODO(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {})
+	test.OK(t, err)
+	defer elock.Unlock()
+
+	retryLock := 200 * time.Millisecond
+
+	start := time.Now()
+	_, _, err = Lock(context.TODO(), repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {})
+	duration := time.Since(start)
+
+	test.Assert(t, err != nil,
+		"create normal lock with exclusively locked repo didn't return an error")
+	test.Assert(t, strings.Contains(err.Error(), "repository is already locked exclusively"),
+		"create normal lock with exclusively locked repo didn't return the correct error")
+	test.Assert(t, retryLock <= duration && duration < retryLock*3/2,
+		"create normal lock with exclusively locked repo didn't wait for the specified timeout")
+}
+
+func TestLockWaitCancel(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, nil)
+
+	elock, _, err := Lock(context.TODO(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {})
+	test.OK(t, err)
+	defer elock.Unlock()
+
+	retryLock := 200 * time.Millisecond
+	cancelAfter := 40 * time.Millisecond
+
+	start := time.Now()
+	ctx, cancel := context.WithCancel(context.TODO())
+	time.AfterFunc(cancelAfter, cancel)
+
+	_, _, err = Lock(ctx, repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {})
+	duration := time.Since(start)
+
+	test.Assert(t, err != nil,
+		"create normal lock with exclusively locked repo didn't return an error")
+	test.Assert(t, strings.Contains(err.Error(), "context canceled"),
+		"create normal lock with exclusively locked repo didn't return the correct error")
+	test.Assert(t, cancelAfter <= duration && duration < retryLock-10*time.Millisecond,
+		"create normal lock with exclusively locked repo didn't return in time, duration %v", duration)
+}
+
+func TestLockWaitSuccess(t *testing.T) {
+	t.Parallel()
+	repo := openLockTestRepo(t, nil)
+
+	elock, _, err := Lock(context.TODO(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {})
+	test.OK(t, err)
+
+	retryLock := 200 * time.Millisecond
+	unlockAfter := 40 * time.Millisecond
+
+	time.AfterFunc(unlockAfter, func() {
+		elock.Unlock()
+	})
+
+	lock, _, err := Lock(context.TODO(), repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {})
+	test.OK(t, err)
+	lock.Unlock()
+}

internal/repository/prune.go

@@ -0,0 +1,638 @@
+package repository
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sort"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/pack"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/progress"
+)
+
+var ErrIndexIncomplete = errors.Fatal("index is not complete")
+var ErrPacksMissing = errors.Fatal("packs from index missing in repo")
+var ErrSizeNotMatching = errors.Fatal("pack size does not match calculated size from index")
+
+// PruneOptions collects all options for the cleanup command.
+type PruneOptions struct {
+	DryRun         bool
+	UnsafeRecovery bool
+
+	MaxUnusedBytes func(used uint64) (unused uint64) // calculates the number of unused bytes after repacking, according to MaxUnused
+	MaxRepackBytes uint64
+
+	RepackCachableOnly bool
+	RepackSmall        bool
+	RepackUncompressed bool
+}
+
+type PruneStats struct {
+	Blobs struct {
+		Used      uint
+		Duplicate uint
+		Unused    uint
+		Remove    uint
+		Repack    uint
+		Repackrm  uint
+	}
+	Size struct {
+		Used         uint64
+		Duplicate    uint64
+		Unused       uint64
+		Remove       uint64
+		Repack       uint64
+		Repackrm     uint64
+		Unref        uint64
+		Uncompressed uint64
+	}
+	Packs struct {
+		Used       uint
+		Unused     uint
+		PartlyUsed uint
+		Unref      uint
+		Keep       uint
+		Repack     uint
+		Remove     uint
+	}
+}
+
+type PrunePlan struct {
+	removePacksFirst restic.IDSet          // packs to remove first (unreferenced packs)
+	repackPacks      restic.IDSet          // packs to repack
+	keepBlobs        restic.CountedBlobSet // blobs to keep during repacking
+	removePacks      restic.IDSet          // packs to remove
+	ignorePacks      restic.IDSet          // packs to ignore when rebuilding the index
+
+	repo  restic.Repository
+	stats PruneStats
+	opts  PruneOptions
+}
+
+type packInfo struct {
+	usedBlobs    uint
+	unusedBlobs  uint
+	usedSize     uint64
+	unusedSize   uint64
+	tpe          restic.BlobType
+	uncompressed bool
+}
+
+type packInfoWithID struct {
+	ID restic.ID
+	packInfo
+	mustCompress bool
+}
+
+// PlanPrune selects which files to rewrite and which to delete and which blobs to keep.
+// Also some summary statistics are returned.
+func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error), printer progress.Printer) (*PrunePlan, error) {
+	var stats PruneStats
+
+	if opts.UnsafeRecovery {
+		// prevent repacking data to make sure users cannot get stuck.
+		opts.MaxRepackBytes = 0
+	}
+	if repo.Connections() < 2 {
+		return nil, fmt.Errorf("prune requires a backend connection limit of at least two")
+	}
+	if repo.Config().Version < 2 && opts.RepackUncompressed {
+		return nil, fmt.Errorf("compression requires at least repository format version 2")
+	}
+
+	usedBlobs, err := getUsedBlobs(ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	printer.P("searching used packs...\n")
+	keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats, printer)
+	if err != nil {
+		return nil, err
+	}
+
+	printer.P("collecting packs for deletion and repacking\n")
+	plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, printer)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(plan.repackPacks) != 0 {
+		blobCount := keepBlobs.Len()
+		// when repacking, we do not want to keep blobs which are
+		// already contained in kept packs, so delete them from keepBlobs
+		err := repo.Index().Each(ctx, func(blob restic.PackedBlob) {
+			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
+				return
+			}
+			keepBlobs.Delete(blob.BlobHandle)
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		if keepBlobs.Len() < blobCount/2 {
+			// replace with copy to shrink map to necessary size if there's a chance to benefit
+			keepBlobs = keepBlobs.Copy()
+		}
+	} else {
+		// keepBlobs is only needed if packs are repacked
+		keepBlobs = nil
+	}
+	plan.keepBlobs = keepBlobs
+
+	plan.repo = repo
+	plan.stats = stats
+	plan.opts = opts
+
+	return &plan, nil
+}
+
+func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *PruneStats, printer progress.Printer) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
+	// iterate over all blobs in index to find out which blobs are duplicates
+	// The counter in usedBlobs describes how many instances of the blob exist in the repository index
+	// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
+	err := idx.Each(ctx, func(blob restic.PackedBlob) {
+		bh := blob.BlobHandle
+		count, ok := usedBlobs[bh]
+		if ok {
+			if count < math.MaxUint8 {
+				// don't overflow, but saturate count at 255
+				// this can lead to a non-optimal pack selection, but won't cause
+				// problems otherwise
+				count++
+			}
+
+			usedBlobs[bh] = count
+		}
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Check if all used blobs have been found in index
+	missingBlobs := restic.NewBlobSet()
+	for bh, count := range usedBlobs {
+		if count == 0 {
+			// blob does not exist in any pack files
+			missingBlobs.Insert(bh)
+		}
+	}
+
+	if len(missingBlobs) != 0 {
+		printer.E("%v not found in the index\n\n"+
+			"Integrity check failed: Data seems to be missing.\n"+
+			"Will not start prune to prevent (additional) data loss!\n"+
+			"Please report this error (along with the output of the 'prune' run) at\n"+
+			"https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
+		return nil, nil, ErrIndexIncomplete
+	}
+
+	indexPack := make(map[restic.ID]packInfo)
+
+	// save computed pack header size
+	sz, err := pack.Size(ctx, idx, true)
+	if err != nil {
+		return nil, nil, err
+	}
+	for pid, hdrSize := range sz {
+		// initialize tpe with NumBlobTypes to indicate it's not set
+		indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
+	}
+
+	hasDuplicates := false
+	// iterate over all blobs in index to generate packInfo
+	err = idx.Each(ctx, func(blob restic.PackedBlob) {
+		ip := indexPack[blob.PackID]
+
+		// Set blob type if not yet set
+		if ip.tpe == restic.NumBlobTypes {
+			ip.tpe = blob.Type
+		}
+
+		// mark mixed packs with "Invalid blob type"
+		if ip.tpe != blob.Type {
+			ip.tpe = restic.InvalidBlob
+		}
+
+		bh := blob.BlobHandle
+		size := uint64(blob.Length)
+		dupCount := usedBlobs[bh]
+		switch {
+		case dupCount >= 2:
+			hasDuplicates = true
+			// mark as unused for now, we will later on select one copy
+			ip.unusedSize += size
+			ip.unusedBlobs++
+
+			// count as duplicate, will later on change one copy to be counted as used
+			stats.Size.Duplicate += size
+			stats.Blobs.Duplicate++
+		case dupCount == 1: // used blob, not duplicate
+			ip.usedSize += size
+			ip.usedBlobs++
+
+			stats.Size.Used += size
+			stats.Blobs.Used++
+		default: // unused blob
+			ip.unusedSize += size
+			ip.unusedBlobs++
+
+			stats.Size.Unused += size
+			stats.Blobs.Unused++
+		}
+		if !blob.IsCompressed() {
+			ip.uncompressed = true
+		}
+		// update indexPack
+		indexPack[blob.PackID] = ip
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// if duplicate blobs exist, those will be set to either "used" or "unused":
+	// - mark only one occurrence of duplicate blobs as used
+	// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
+	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
+	if hasDuplicates {
+		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
+		err = idx.Each(ctx, func(blob restic.PackedBlob) {
+			bh := blob.BlobHandle
+			count, ok := usedBlobs[bh]
+			// skip non-duplicate, aka. normal blobs
+			// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
+			if !ok || count == 1 {
+				return
+			}
+
+			ip := indexPack[blob.PackID]
+			size := uint64(blob.Length)
+			switch {
+			case ip.usedBlobs > 0, count == 0:
+				// other used blobs in pack or "last" occurrence ->  transition to used
+				ip.usedSize += size
+				ip.usedBlobs++
+				ip.unusedSize -= size
+				ip.unusedBlobs--
+				// same for the global statistics
+				stats.Size.Used += size
+				stats.Blobs.Used++
+				stats.Size.Duplicate -= size
+				stats.Blobs.Duplicate--
+				// let other occurrences remain marked as unused
+				usedBlobs[bh] = 1
+			default:
+				// remain unused and decrease counter
+				count--
+				if count == 1 {
+					// setting count to 1 would lead to forgetting that this blob had duplicates
+					// thus use the special value zero. This will select the last instance of the blob for keeping.
+					count = 0
+				}
+				usedBlobs[bh] = count
+			}
+			// update indexPack
+			indexPack[blob.PackID] = ip
+		})
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// Sanity check. If no duplicates exist, all blobs have value 1. After handling
+	// duplicates, this also applies to duplicates.
+	for _, count := range usedBlobs {
+		if count != 1 {
+			panic("internal error during blob selection")
+		}
+	}
+
+	return usedBlobs, indexPack, nil
+}
+
+func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *PruneStats, printer progress.Printer) (PrunePlan, error) {
+	removePacksFirst := restic.NewIDSet()
+	removePacks := restic.NewIDSet()
+	repackPacks := restic.NewIDSet()
+
+	var repackCandidates []packInfoWithID
+	var repackSmallCandidates []packInfoWithID
+	repoVersion := repo.Config().Version
+	// only repack very small files by default
+	targetPackSize := repo.PackSize() / 25
+	if opts.RepackSmall {
+		// consider files with at least 80% of the target size as large enough
+		targetPackSize = repo.PackSize() / 5 * 4
+	}
+
+	// loop over all packs and decide what to do
+	bar := printer.NewCounter("packs processed")
+	bar.SetMax(uint64(len(indexPack)))
+	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
+		p, ok := indexPack[id]
+		if !ok {
+			// Pack was not referenced in index and is not used  => immediately remove!
+			printer.V("will remove pack %v as it is unused and not indexed\n", id.Str())
+			removePacksFirst.Insert(id)
+			stats.Size.Unref += uint64(packSize)
+			return nil
+		}
+
+		if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
+			// Pack size does not fit and pack is needed => error
+			// If the pack is not needed, this is no error, the pack can
+			// and will be simply removed, see below.
+			printer.E("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
+				id.Str(), p.unusedSize+p.usedSize, packSize)
+			return ErrSizeNotMatching
+		}
+
+		// statistics
+		switch {
+		case p.usedBlobs == 0:
+			stats.Packs.Unused++
+		case p.unusedBlobs == 0:
+			stats.Packs.Used++
+		default:
+			stats.Packs.PartlyUsed++
+		}
+
+		if p.uncompressed {
+			stats.Size.Uncompressed += p.unusedSize + p.usedSize
+		}
+		mustCompress := false
+		if repoVersion >= 2 {
+			// repo v2: always repack tree blobs if uncompressed
+			// compress data blobs if requested
+			mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
+		}
+
+		// decide what to do
+		switch {
+		case p.usedBlobs == 0:
+			// All blobs in pack are no longer used => remove pack!
+			removePacks.Insert(id)
+			stats.Blobs.Remove += p.unusedBlobs
+			stats.Size.Remove += p.unusedSize
+
+		case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
+			// if this is a data pack and --repack-cacheable-only is set => keep pack!
+			stats.Packs.Keep++
+
+		case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
+			if packSize >= int64(targetPackSize) {
+				// All blobs in pack are used and not mixed => keep pack!
+				stats.Packs.Keep++
+			} else {
+				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
+			}
+
+		default:
+			// all other packs are candidates for repacking
+			repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
+		}
+
+		delete(indexPack, id)
+		bar.Add(1)
+		return nil
+	})
+	bar.Done()
+	if err != nil {
+		return PrunePlan{}, err
+	}
+
+	// At this point indexPacks contains only missing packs!
+
+	// missing packs that are not needed can be ignored
+	ignorePacks := restic.NewIDSet()
+	for id, p := range indexPack {
+		if p.usedBlobs == 0 {
+			ignorePacks.Insert(id)
+			stats.Blobs.Remove += p.unusedBlobs
+			stats.Size.Remove += p.unusedSize
+			delete(indexPack, id)
+		}
+	}
+
+	if len(indexPack) != 0 {
+		printer.E("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
+		for id := range indexPack {
+			printer.E("  %v\n", id)
+		}
+		return PrunePlan{}, ErrPacksMissing
+	}
+	if len(ignorePacks) != 0 {
+		printer.E("Missing but unneeded pack files are referenced in the index, will be repaired\n")
+		for id := range ignorePacks {
+			printer.E("will forget missing pack file %v\n", id)
+		}
+	}
+
+	if len(repackSmallCandidates) < 10 {
+		// too few small files to be worth the trouble, this also prevents endlessly repacking
+		// if there is just a single pack file below the target size
+		stats.Packs.Keep += uint(len(repackSmallCandidates))
+	} else {
+		repackCandidates = append(repackCandidates, repackSmallCandidates...)
+	}
+
+	// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
+	// This is equivalent to sorting by unused / total space.
+	// Instead of unused[i] / used[i] > unused[j] / used[j] we use
+	// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
+	// Moreover packs containing trees and too small packs are sorted to the beginning
+	sort.Slice(repackCandidates, func(i, j int) bool {
+		pi := repackCandidates[i].packInfo
+		pj := repackCandidates[j].packInfo
+		switch {
+		case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
+			return true
+		case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
+			return false
+		case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
+			return true
+		case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
+			return false
+		}
+		return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
+	})
+
+	repack := func(id restic.ID, p packInfo) {
+		repackPacks.Insert(id)
+		stats.Blobs.Repack += p.unusedBlobs + p.usedBlobs
+		stats.Size.Repack += p.unusedSize + p.usedSize
+		stats.Blobs.Repackrm += p.unusedBlobs
+		stats.Size.Repackrm += p.unusedSize
+		if p.uncompressed {
+			stats.Size.Uncompressed -= p.unusedSize + p.usedSize
+		}
+	}
+
+	// calculate limit for number of unused bytes in the repo after repacking
+	maxUnusedSizeAfter := opts.MaxUnusedBytes(stats.Size.Used)
+
+	for _, p := range repackCandidates {
+		reachedUnusedSizeAfter := (stats.Size.Unused-stats.Size.Remove-stats.Size.Repackrm < maxUnusedSizeAfter)
+		reachedRepackSize := stats.Size.Repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
+		packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
+
+		switch {
+		case reachedRepackSize:
+			stats.Packs.Keep++
+
+		case p.tpe != restic.DataBlob, p.mustCompress:
+			// repacking non-data packs / uncompressed-trees is only limited by repackSize
+			repack(p.ID, p.packInfo)
+
+		case reachedUnusedSizeAfter && packIsLargeEnough:
+			// for all other packs stop repacking if tolerated unused size is reached.
+			stats.Packs.Keep++
+
+		default:
+			repack(p.ID, p.packInfo)
+		}
+	}
+
+	stats.Packs.Unref = uint(len(removePacksFirst))
+	stats.Packs.Repack = uint(len(repackPacks))
+	stats.Packs.Remove = uint(len(removePacks))
+
+	if repo.Config().Version < 2 {
+		// compression not supported for repository format version 1
+		stats.Size.Uncompressed = 0
+	}
+
+	return PrunePlan{removePacksFirst: removePacksFirst,
+		removePacks: removePacks,
+		repackPacks: repackPacks,
+		ignorePacks: ignorePacks,
+	}, nil
+}
+
+func (plan *PrunePlan) Stats() PruneStats {
+	return plan.stats
+}
+
+// Execute does the actual pruning:
+// - remove unreferenced packs first
+// - repack given pack files while keeping the given blobs
+// - rebuild the index while ignoring all files that will be deleted
+// - delete the files
+// plan.removePacks and plan.ignorePacks are modified in this function.
+func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) (err error) {
+	if plan.opts.DryRun {
+		printer.V("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
+		if len(plan.removePacksFirst) > 0 {
+			printer.V("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
+		}
+		printer.V("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
+		printer.V("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
+		// Always quit here if DryRun was set!
+		return nil
+	}
+
+	repo := plan.repo
+	// make sure the plan can only be used once
+	plan.repo = nil
+
+	// unreferenced packs can be safely deleted first
+	if len(plan.removePacksFirst) != 0 {
+		printer.P("deleting unreferenced packs\n")
+		_ = deleteFiles(ctx, true, repo, plan.removePacksFirst, restic.PackFile, printer)
+	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	if len(plan.repackPacks) != 0 {
+		printer.P("repacking packs\n")
+		bar := printer.NewCounter("packs repacked")
+		bar.SetMax(uint64(len(plan.repackPacks)))
+		_, err := Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
+		bar.Done()
+		if err != nil {
+			return errors.Fatal(err.Error())
+		}
+
+		// Also remove repacked packs
+		plan.removePacks.Merge(plan.repackPacks)
+
+		if len(plan.keepBlobs) != 0 {
+			printer.E("%v was not repacked\n\n"+
+				"Integrity check failed.\n"+
+				"Please report this error (along with the output of the 'prune' run) at\n"+
+				"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
+			return errors.Fatal("internal error: blobs were not repacked")
+		}
+
+		// allow GC of the blob set
+		plan.keepBlobs = nil
+	}
+
+	if len(plan.ignorePacks) == 0 {
+		plan.ignorePacks = plan.removePacks
+	} else {
+		plan.ignorePacks.Merge(plan.removePacks)
+	}
+
+	if plan.opts.UnsafeRecovery {
+		printer.P("deleting index files\n")
+		indexFiles := repo.Index().(*index.MasterIndex).IDs()
+		err = deleteFiles(ctx, false, repo, indexFiles, restic.IndexFile, printer)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+	} else if len(plan.ignorePacks) != 0 {
+		err = rebuildIndexFiles(ctx, repo, plan.ignorePacks, nil, false, printer)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+	}
+
+	if len(plan.removePacks) != 0 {
+		printer.P("removing %d old packs\n", len(plan.removePacks))
+		_ = deleteFiles(ctx, true, repo, plan.removePacks, restic.PackFile, printer)
+	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	if plan.opts.UnsafeRecovery {
+		err = rebuildIndexFiles(ctx, repo, plan.ignorePacks, nil, true, printer)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+	}
+
+	if err != nil {
+		return err
+	}
+
+	// drop outdated in-memory index
+	repo.ClearIndex()
+
+	printer.P("done\n")
+	return nil
+}
+
+// deleteFiles deletes the given fileList of fileType in parallel
+// if ignoreError=true, it will print a warning if there was an error, else it will abort.
+func deleteFiles(ctx context.Context, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType, printer progress.Printer) error {
+	bar := printer.NewCounter("files deleted")
+	defer bar.Done()
+
+	return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error {
+		if err != nil {
+			printer.E("unable to remove %v/%v from the repository\n", fileType, id)
+			if !ignoreError {
+				return err
+			}
+		}
+		printer.VV("removed %v/%v\n", fileType, id)
+		return nil
+	}, bar)
+}

internal/repository/prune_test.go

@@ -0,0 +1,105 @@
+package repository_test
+
+import (
+	"context"
+	"math"
+	"testing"
+
+	"github.com/restic/restic/internal/checker"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/progress"
+	"golang.org/x/sync/errgroup"
+)
+
+func testPrune(t *testing.T, opts repository.PruneOptions, errOnUnused bool) {
+	repo := repository.TestRepository(t).(*repository.Repository)
+	createRandomBlobs(t, repo, 4, 0.5, true)
+	createRandomBlobs(t, repo, 5, 0.5, true)
+	keep, _ := selectBlobs(t, repo, 0.5)
+
+	var wg errgroup.Group
+	repo.StartPackUploader(context.TODO(), &wg)
+	// duplicate a few blobs to exercise those code paths
+	for blob := range keep {
+		buf, err := repo.LoadBlob(context.TODO(), blob.Type, blob.ID, nil)
+		rtest.OK(t, err)
+		_, _, _, err = repo.SaveBlob(context.TODO(), blob.Type, buf, blob.ID, true)
+		rtest.OK(t, err)
+	}
+	rtest.OK(t, repo.Flush(context.TODO()))
+
+	plan, err := repository.PlanPrune(context.TODO(), opts, repo, func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error) {
+		return restic.NewCountedBlobSet(keep.List()...), nil
+	}, &progress.NoopPrinter{})
+	rtest.OK(t, err)
+
+	rtest.OK(t, plan.Execute(context.TODO(), &progress.NoopPrinter{}))
+
+	repo = repository.TestOpenBackend(t, repo.Backend()).(*repository.Repository)
+	checker.TestCheckRepo(t, repo, true)
+
+	if errOnUnused {
+		existing := listBlobs(repo)
+		rtest.Assert(t, existing.Equals(keep), "unexpected blobs, wanted %v got %v", keep, existing)
+	}
+}
+
+func TestPrune(t *testing.T) {
+	for _, test := range []struct {
+		name        string
+		opts        repository.PruneOptions
+		errOnUnused bool
+	}{
+		{
+			name: "0",
+			opts: repository.PruneOptions{
+				MaxRepackBytes: math.MaxUint64,
+				MaxUnusedBytes: func(used uint64) (unused uint64) { return 0 },
+			},
+			errOnUnused: true,
+		},
+		{
+			name: "50",
+			opts: repository.PruneOptions{
+				MaxRepackBytes: math.MaxUint64,
+				MaxUnusedBytes: func(used uint64) (unused uint64) { return used / 2 },
+			},
+		},
+		{
+			name: "unlimited",
+			opts: repository.PruneOptions{
+				MaxRepackBytes: math.MaxUint64,
+				MaxUnusedBytes: func(used uint64) (unused uint64) { return math.MaxUint64 },
+			},
+		},
+		{
+			name: "cachableonly",
+			opts: repository.PruneOptions{
+				MaxRepackBytes:     math.MaxUint64,
+				MaxUnusedBytes:     func(used uint64) (unused uint64) { return used / 20 },
+				RepackCachableOnly: true,
+			},
+		},
+		{
+			name: "small",
+			opts: repository.PruneOptions{
+				MaxRepackBytes: math.MaxUint64,
+				MaxUnusedBytes: func(used uint64) (unused uint64) { return math.MaxUint64 },
+				RepackSmall:    true,
+			},
+			errOnUnused: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			testPrune(t, test.opts, test.errOnUnused)
+		})
+		t.Run(test.name+"-recovery", func(t *testing.T) {
+			opts := test.opts
+			opts.UnsafeRecovery = true
+			// unsafeNoSpaceRecovery does not repack partially used pack files
+			testPrune(t, opts, false)
+		})
+	}
+}

internal/repository/repack.go

@@ -72,7 +72,7 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito
 				return wgCtx.Err()
 			}
 		}
-		return nil
+		return wgCtx.Err()
 	})
 
 	worker := func() error {

internal/repository/repack_test.go

@@ -18,7 +18,7 @@ func randomSize(min, max int) int {
 	return rand.Intn(max-min) + min
 }
 
-func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32) {
+func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32, smallBlobs bool) {
 	var wg errgroup.Group
 	repo.StartPackUploader(context.TODO(), &wg)
 
@@ -30,7 +30,11 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl
 
 		if rand.Float32() < pData {
 			tpe = restic.DataBlob
-			length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data
+			if smallBlobs {
+				length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB of data
+			} else {
+				length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data
+			}
 		} else {
 			tpe = restic.TreeBlob
 			length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB
@@ -121,8 +125,12 @@ func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2
 }
 
 func listPacks(t *testing.T, repo restic.Lister) restic.IDSet {
+	return listFiles(t, repo, restic.PackFile)
+}
+
+func listFiles(t *testing.T, repo restic.Lister, tpe backend.FileType) restic.IDSet {
 	list := restic.NewIDSet()
-	err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
+	err := repo.List(context.TODO(), tpe, func(id restic.ID, size int64) error {
 		list.Insert(id)
 		return nil
 	})
@@ -166,12 +174,6 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest
 	}
 }
 
-func flush(t *testing.T, repo restic.Repository) {
-	if err := repo.Flush(context.TODO()); err != nil {
-		t.Fatalf("repo.SaveIndex() %v", err)
-	}
-}
-
 func rebuildIndex(t *testing.T, repo restic.Repository) {
 	err := repo.SetIndex(index.NewMasterIndex())
 	rtest.OK(t, err)
@@ -219,7 +221,9 @@ func testRepack(t *testing.T, version uint) {
 	rand.Seed(seed)
 	t.Logf("rand seed is %v", seed)
 
-	createRandomBlobs(t, repo, 100, 0.7)
+	// add a small amount of blobs twice to create multiple pack files
+	createRandomBlobs(t, repo, 10, 0.7, false)
+	createRandomBlobs(t, repo, 10, 0.7, false)
 
 	packsBefore := listPacks(t, repo)
 
@@ -233,8 +237,6 @@ func testRepack(t *testing.T, version uint) {
 			packsBefore, packsAfter)
 	}
 
-	flush(t, repo)
-
 	removeBlobs, keepBlobs := selectBlobs(t, repo, 0.2)
 
 	removePacks := findPacksForBlobs(t, repo, removeBlobs)
@@ -302,8 +304,9 @@ func testRepackCopy(t *testing.T, version uint) {
 	rand.Seed(seed)
 	t.Logf("rand seed is %v", seed)
 
-	createRandomBlobs(t, repo, 100, 0.7)
-	flush(t, repo)
+	// add a small amount of blobs twice to create multiple pack files
+	createRandomBlobs(t, repo, 10, 0.7, false)
+	createRandomBlobs(t, repo, 10, 0.7, false)
 
 	_, keepBlobs := selectBlobs(t, repo, 0.2)
 	copyPacks := findPacksForBlobs(t, repo, keepBlobs)
@@ -343,7 +346,7 @@ func testRepackWrongBlob(t *testing.T, version uint) {
 	rand.Seed(seed)
 	t.Logf("rand seed is %v", seed)
 
-	createRandomBlobs(t, repo, 5, 0.7)
+	createRandomBlobs(t, repo, 5, 0.7, false)
 	createRandomWrongBlob(t, repo)
 
 	// just keep all blobs, but also rewrite every pack

internal/repository/repair_index.go

@@ -0,0 +1,132 @@
+package repository
+
+import (
+	"context"
+
+	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/pack"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/progress"
+)
+
+type RepairIndexOptions struct {
+	ReadAllPacks bool
+}
+
+func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions, printer progress.Printer) error {
+	var obsoleteIndexes restic.IDs
+	packSizeFromList := make(map[restic.ID]int64)
+	packSizeFromIndex := make(map[restic.ID]int64)
+	removePacks := restic.NewIDSet()
+
+	if opts.ReadAllPacks {
+		// get list of old index files but start with empty index
+		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
+			obsoleteIndexes = append(obsoleteIndexes, id)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		printer.P("loading indexes...\n")
+		mi := index.NewMasterIndex()
+		err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error {
+			if err != nil {
+				printer.E("removing invalid index %v: %v\n", id, err)
+				obsoleteIndexes = append(obsoleteIndexes, id)
+				return nil
+			}
+
+			mi.Insert(idx)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		err = mi.MergeFinalIndexes()
+		if err != nil {
+			return err
+		}
+
+		err = repo.SetIndex(mi)
+		if err != nil {
+			return err
+		}
+		packSizeFromIndex, err = pack.Size(ctx, repo.Index(), false)
+		if err != nil {
+			return err
+		}
+	}
+
+	printer.P("getting pack files to read...\n")
+	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
+		size, ok := packSizeFromIndex[id]
+		if !ok || size != packSize {
+			// Pack was not referenced in index or size does not match
+			packSizeFromList[id] = packSize
+			removePacks.Insert(id)
+		}
+		if !ok {
+			printer.E("adding pack file to index %v\n", id)
+		} else if size != packSize {
+			printer.E("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
+		}
+		delete(packSizeFromIndex, id)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	for id := range packSizeFromIndex {
+		// forget pack files that are referenced in the index but do not exist
+		// when rebuilding the index
+		removePacks.Insert(id)
+		printer.E("removing not found pack file %v\n", id)
+	}
+
+	if len(packSizeFromList) > 0 {
+		printer.P("reading pack files\n")
+		bar := printer.NewCounter("packs")
+		bar.SetMax(uint64(len(packSizeFromList)))
+		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
+		bar.Done()
+		if err != nil {
+			return err
+		}
+
+		for _, id := range invalidFiles {
+			printer.V("skipped incomplete pack file: %v\n", id)
+		}
+	}
+
+	err = rebuildIndexFiles(ctx, repo, removePacks, obsoleteIndexes, false, printer)
+	if err != nil {
+		return err
+	}
+
+	// drop outdated in-memory index
+	repo.ClearIndex()
+	return nil
+}
+
+func rebuildIndexFiles(ctx context.Context, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool, printer progress.Printer) error {
+	printer.P("rebuilding index\n")
+
+	bar := printer.NewCounter("packs processed")
+	return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
+		SaveProgress: bar,
+		DeleteProgress: func() *progress.Counter {
+			return printer.NewCounter("old indexes deleted")
+		},
+		DeleteReport: func(id restic.ID, err error) {
+			if err != nil {
+				printer.VV("failed to remove index %v: %v\n", id.String(), err)
+			} else {
+				printer.VV("removed index %v\n", id.String())
+			}
+		},
+		SkipDeletion: skipDeletion,
+	})
+}

internal/repository/repair_index_test.go

@@ -0,0 +1,79 @@
+package repository_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/checker"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/progress"
+)
+
+func listIndex(t *testing.T, repo restic.Lister) restic.IDSet {
+	return listFiles(t, repo, restic.IndexFile)
+}
+
+func testRebuildIndex(t *testing.T, readAllPacks bool, damage func(t *testing.T, repo *repository.Repository)) {
+	repo := repository.TestRepository(t).(*repository.Repository)
+	createRandomBlobs(t, repo, 4, 0.5, true)
+	createRandomBlobs(t, repo, 5, 0.5, true)
+	indexes := listIndex(t, repo)
+	t.Logf("old indexes %v", indexes)
+
+	damage(t, repo)
+
+	repo = repository.TestOpenBackend(t, repo.Backend()).(*repository.Repository)
+	rtest.OK(t, repository.RepairIndex(context.TODO(), repo, repository.RepairIndexOptions{
+		ReadAllPacks: readAllPacks,
+	}, &progress.NoopPrinter{}))
+
+	newIndexes := listIndex(t, repo)
+	old := indexes.Intersect(newIndexes)
+	rtest.Assert(t, len(old) == 0, "expected old indexes to be removed, found %v", old)
+
+	checker.TestCheckRepo(t, repo, true)
+}
+
+func TestRebuildIndex(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		damage func(t *testing.T, repo *repository.Repository)
+	}{
+		{
+			"valid index",
+			func(t *testing.T, repo *repository.Repository) {},
+		},
+		{
+			"damaged index",
+			func(t *testing.T, repo *repository.Repository) {
+				index := listIndex(t, repo).List()[0]
+				replaceFile(t, repo, backend.Handle{Type: restic.IndexFile, Name: index.String()}, func(b []byte) []byte {
+					b[0] ^= 0xff
+					return b
+				})
+			},
+		},
+		{
+			"missing index",
+			func(t *testing.T, repo *repository.Repository) {
+				index := listIndex(t, repo).List()[0]
+				rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: index.String()}))
+			},
+		},
+		{
+			"missing pack",
+			func(t *testing.T, repo *repository.Repository) {
+				pack := listPacks(t, repo).List()[0]
+				rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: pack.String()}))
+			},
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			testRebuildIndex(t, false, test.damage)
+			testRebuildIndex(t, true, test.damage)
+		})
+	}
+}

internal/repository/repair_pack.go

@@ -60,19 +60,7 @@ func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet,
 	}
 
 	// remove salvaged packs from index
-	printer.P("rebuilding index")
-
-	bar = printer.NewCounter("packs processed")
-	err = repo.Index().Save(ctx, repo, ids, nil, restic.MasterIndexSaveOpts{
-		SaveProgress: bar,
-		DeleteProgress: func() *progress.Counter {
-			return printer.NewCounter("old indexes deleted")
-		},
-		DeleteReport: func(id restic.ID, _ error) {
-			printer.VV("removed index %v", id.String())
-		},
-	})
-
+	err = rebuildIndexFiles(ctx, repo, ids, nil, false, printer)
 	if err != nil {
 		return err
 	}

internal/repository/repair_pack_test.go

@@ -17,7 +17,7 @@ import (
 
 func listBlobs(repo restic.Repository) restic.BlobSet {
 	blobs := restic.NewBlobSet()
-	repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
+	_ = repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
 		blobs.Insert(pb.BlobHandle)
 	})
 	return blobs
@@ -109,7 +109,7 @@ func testRepairBrokenPack(t *testing.T, version uint) {
 			rand.Seed(seed)
 			t.Logf("rand seed is %v", seed)
 
-			createRandomBlobs(t, repo, 5, 0.7)
+			createRandomBlobs(t, repo, 5, 0.7, true)
 			packsBefore := listPacks(t, repo)
 			blobsBefore := listBlobs(repo)
 

internal/repository/repository.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"runtime"
 	"sort"
@@ -142,9 +143,6 @@ func (r *Repository) DisableAutoIndexUpdate() {
 // setConfig assigns the given config and updates the repository parameters accordingly
 func (r *Repository) setConfig(cfg restic.Config) {
 	r.cfg = cfg
-	if r.cfg.Version >= 2 {
-		r.idx.MarkCompressed()
-	}
 }
 
 // Config returns the repository configuration.
@@ -637,9 +635,21 @@ func (r *Repository) Index() restic.MasterIndex {
 // SetIndex instructs the repository to use the given index.
 func (r *Repository) SetIndex(i restic.MasterIndex) error {
 	r.idx = i.(*index.MasterIndex)
+	r.configureIndex()
 	return r.prepareCache()
 }
 
+func (r *Repository) ClearIndex() {
+	r.idx = index.NewMasterIndex()
+	r.configureIndex()
+}
+
+func (r *Repository) configureIndex() {
+	if r.cfg.Version >= 2 {
+		r.idx.MarkCompressed()
+	}
+}
+
 // LoadIndex loads all index files from the backend in parallel and stores them
 func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
 	debug.Log("Loading index")
@@ -662,6 +672,9 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
 		defer p.Done()
 	}
 
+	// reset in-memory index before loading it from the repository
+	r.ClearIndex()
+
 	err = index.ForAllIndexes(ctx, indexList, r, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
 		if err != nil {
 			return err
@@ -691,15 +704,21 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
 		defer cancel()
 
 		invalidIndex := false
-		r.idx.Each(ctx, func(blob restic.PackedBlob) {
+		err := r.idx.Each(ctx, func(blob restic.PackedBlob) {
 			if blob.IsCompressed() {
 				invalidIndex = true
 			}
 		})
+		if err != nil {
+			return err
+		}
 		if invalidIndex {
 			return errors.New("index uses feature not supported by repository version 1")
 		}
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	// remove index files from the cache which have been removed in the repo
 	return r.prepareCache()
@@ -917,6 +936,10 @@ func (r *Repository) Close() error {
 // occupies in the repo (compressed or not, including encryption overhead).
 func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) {
 
+	if int64(len(buf)) > math.MaxUint32 {
+		return restic.ID{}, false, 0, fmt.Errorf("blob is larger than 4GB")
+	}
+
 	// compute plaintext hash if not already set
 	if id.IsNull() {
 		// Special case the hash calculation for all zero chunks. This is especially

internal/repository/repository_test.go

@@ -221,10 +221,9 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
 var repoFixture = filepath.Join("testdata", "test-repo.tar.gz")
 
 func TestRepositoryLoadIndex(t *testing.T) {
-	repodir, cleanup := rtest.Env(t, repoFixture)
+	repo, cleanup := repository.TestFromFixture(t, repoFixture)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
 	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
 }
 
@@ -243,8 +242,7 @@ func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*
 }
 
 func TestRepositoryLoadUnpackedBroken(t *testing.T) {
-	repodir, cleanup := rtest.Env(t, repoFixture)
-	defer cleanup()
+	repo := repository.TestRepository(t)
 
 	data := rtest.Random(23, 12345)
 	id := restic.Hash(data)
@@ -252,9 +250,8 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
 	// damage buffer
 	data[0] ^= 0xff
 
-	repo := repository.TestOpenLocal(t, repodir)
 	// store broken file
-	err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, nil))
+	err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, repo.Backend().Hasher()))
 	rtest.OK(t, err)
 
 	// without a retry backend this will just return an error that the file is broken
@@ -289,10 +286,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
 
 	be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2})
 	rtest.OK(t, err)
-	repo, err := repository.New(&damageOnceBackend{Backend: be}, repository.Options{})
-	rtest.OK(t, err)
-	err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "")
-	rtest.OK(t, err)
+	repo := repository.TestOpenBackend(t, &damageOnceBackend{Backend: be})
 
 	rtest.OK(t, repo.LoadIndex(context.TODO(), nil))
 }
@@ -376,13 +370,13 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
 		idx, err := loadIndex(context.TODO(), repo, id)
 		rtest.OK(t, err)
 
-		idx.Each(context.TODO(), func(pb restic.PackedBlob) {
+		rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) {
 			if _, ok := packEntries[pb.PackID]; !ok {
 				packEntries[pb.PackID] = make(map[restic.ID]struct{})
 			}
 
 			packEntries[pb.PackID][id] = struct{}{}
-		})
+		}))
 		return nil
 	})
 	if err != nil {

internal/repository/testing.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sync"
 	"testing"
 
 	"github.com/restic/restic/internal/backend"
@@ -17,21 +18,22 @@ import (
 	"github.com/restic/chunker"
 )
 
-// testKDFParams are the parameters for the KDF to be used during testing.
-var testKDFParams = crypto.Params{
-	N: 128,
-	R: 1,
-	P: 1,
-}
-
 type logger interface {
 	Logf(format string, args ...interface{})
 }
 
+var paramsOnce sync.Once
+
 // TestUseLowSecurityKDFParameters configures low-security KDF parameters for testing.
 func TestUseLowSecurityKDFParameters(t logger) {
 	t.Logf("using low-security KDF parameters for test")
-	Params = &testKDFParams
+	paramsOnce.Do(func() {
+		params = &crypto.Params{
+			N: 128,
+			R: 1,
+			P: 1,
+		}
+	})
 }
 
 // TestBackend returns a fully configured in-memory backend.
@@ -39,7 +41,7 @@ func TestBackend(_ testing.TB) backend.Backend {
 	return mem.New()
 }
 
-const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
+const testChunkerPol = chunker.Pol(0x3DA3358B4DC173)
 
 // TestRepositoryWithBackend returns a repository initialized with a test
 // password. If be is nil, an in-memory backend is used. A constant polynomial
@@ -58,8 +60,11 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint, o
 		t.Fatalf("TestRepository(): new repo failed: %v", err)
 	}
 
-	cfg := restic.TestCreateConfig(t, TestChunkerPol, version)
-	err = repo.init(context.TODO(), test.TestPassword, cfg)
+	if version == 0 {
+		version = restic.StableRepoVersion
+	}
+	pol := testChunkerPol
+	err = repo.Init(context.TODO(), version, test.TestPassword, &pol)
 	if err != nil {
 		t.Fatalf("TestRepository(): initialize repo failed: %v", err)
 	}
@@ -98,8 +103,15 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
 	return TestRepositoryWithBackend(t, nil, version, opts)
 }
 
+func TestFromFixture(t testing.TB, repoFixture string) (restic.Repository, func()) {
+	repodir, cleanup := test.Env(t, repoFixture)
+	repo := TestOpenLocal(t, repodir)
+
+	return repo, cleanup
+}
+
 // TestOpenLocal opens a local repository.
-func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
+func TestOpenLocal(t testing.TB, dir string) restic.Repository {
 	var be backend.Backend
 	be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2})
 	if err != nil {
@@ -108,6 +120,10 @@ func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
 
 	be = retry.New(be, 3, nil, nil)
 
+	return TestOpenBackend(t, be)
+}
+
+func TestOpenBackend(t testing.TB, be backend.Backend) restic.Repository {
 	repo, err := New(be, Options{})
 	if err != nil {
 		t.Fatal(err)

internal/restic/config.go

@@ -2,6 +2,7 @@ package restic
 
 import (
 	"context"
+	"sync"
 	"testing"
 
 	"github.com/restic/restic/internal/errors"
@@ -50,29 +51,16 @@ func CreateConfig(version uint) (Config, error) {
 	return cfg, nil
 }
 
-// TestCreateConfig creates a config for use within tests.
-func TestCreateConfig(t testing.TB, pol chunker.Pol, version uint) (cfg Config) {
-	cfg.ChunkerPolynomial = pol
-
-	cfg.ID = NewRandomID().String()
-	if version == 0 {
-		version = StableRepoVersion
-	}
-	if version < MinRepoVersion || version > MaxRepoVersion {
-		t.Fatalf("version %d is out of range", version)
-	}
-	cfg.Version = version
-
-	return cfg
-}
-
 var checkPolynomial = true
+var checkPolynomialOnce sync.Once
 
 // TestDisableCheckPolynomial disables the check that the polynomial used for
 // the chunker.
 func TestDisableCheckPolynomial(t testing.TB) {
 	t.Logf("disabling check of the chunker polynomial")
-	checkPolynomial = false
+	checkPolynomialOnce.Do(func() {
+		checkPolynomial = false
+	})
 }
 
 // LoadConfig returns loads, checks and returns the config for a repository.

internal/restic/node.go

@@ -84,7 +84,7 @@ type Node struct {
 	User       string      `json:"user,omitempty"`
 	Group      string      `json:"group,omitempty"`
 	Inode      uint64      `json:"inode,omitempty"`
-	DeviceID   uint64      `json:"device_id,omitempty"` // device id of the file, stat.st_dev
+	DeviceID   uint64      `json:"device_id,omitempty"` // device id of the file, stat.st_dev, only stored for hardlinks
 	Size       uint64      `json:"size,omitempty"`
 	Links      uint64      `json:"links,omitempty"`
 	LinkTarget string      `json:"linktarget,omitempty"`
@@ -136,7 +136,7 @@ func (node Node) String() string {
 
 // NodeFromFileInfo returns a new node from the given path and FileInfo. It
 // returns the first error that is encountered, together with a node.
-func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
+func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*Node, error) {
 	mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
 	node := &Node{
 		Path:    path,
@@ -150,7 +150,7 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
 		node.Size = uint64(fi.Size())
 	}
 
-	err := node.fillExtra(path, fi)
+	err := node.fillExtra(path, fi, ignoreXattrListError)
 	return node, err
 }
 
@@ -677,7 +677,7 @@ func lookupGroup(gid uint32) string {
 	return group
 }
 
-func (node *Node) fillExtra(path string, fi os.FileInfo) error {
+func (node *Node) fillExtra(path string, fi os.FileInfo, ignoreXattrListError bool) error {
 	stat, ok := toStatT(fi.Sys())
 	if !ok {
 		// fill minimal info with current values for uid, gid
@@ -726,10 +726,13 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
 	return err
 }
 
-func (node *Node) fillExtendedAttributes(path string) error {
+func (node *Node) fillExtendedAttributes(path string, ignoreListError bool) error {
 	xattrs, err := Listxattr(path)
 	debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err)
 	if err != nil {
+		if ignoreListError && IsListxattrPermissionError(err) {
+			return nil
+		}
 		return err
 	}
 

internal/restic/node_aix.go

@@ -33,6 +33,10 @@ func Listxattr(path string) ([]string, error) {
 	return nil, nil
 }
 
+func IsListxattrPermissionError(_ error) bool {
+	return false
+}
+
 // Setxattr is a no-op on AIX.
 func Setxattr(path, name string, data []byte) error {
 	return nil

internal/restic/node_netbsd.go

@@ -23,6 +23,10 @@ func Listxattr(path string) ([]string, error) {
 	return nil, nil
 }
 
+func IsListxattrPermissionError(_ error) bool {
+	return false
+}
+
 // Setxattr is a no-op on netbsd.
 func Setxattr(path, name string, data []byte) error {
 	return nil

internal/restic/node_openbsd.go

@@ -23,6 +23,10 @@ func Listxattr(path string) ([]string, error) {
 	return nil, nil
 }
 
+func IsListxattrPermissionError(_ error) bool {
+	return false
+}
+
 // Setxattr is a no-op on openbsd.
 func Setxattr(path, name string, data []byte) error {
 	return nil

internal/restic/node_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/test"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -31,7 +32,7 @@ func BenchmarkNodeFillUser(t *testing.B) {
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		_, err := NodeFromFileInfo(path, fi)
+		_, err := NodeFromFileInfo(path, fi, false)
 		rtest.OK(t, err)
 	}
 
@@ -55,7 +56,7 @@ func BenchmarkNodeFromFileInfo(t *testing.B) {
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		_, err := NodeFromFileInfo(path, fi)
+		_, err := NodeFromFileInfo(path, fi, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -227,8 +228,11 @@ func TestNodeRestoreAt(t *testing.T) {
 			fi, err := os.Lstat(nodePath)
 			rtest.OK(t, err)
 
-			n2, err := NodeFromFileInfo(nodePath, fi)
+			n2, err := NodeFromFileInfo(nodePath, fi, false)
 			rtest.OK(t, err)
+			n3, err := NodeFromFileInfo(nodePath, fi, true)
+			rtest.OK(t, err)
+			rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3))
 
 			rtest.Assert(t, test.Name == n2.Name,
 				"%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name)

internal/restic/node_unix_test.go

@@ -128,7 +128,7 @@ func TestNodeFromFileInfo(t *testing.T) {
 				return
 			}
 
-			node, err := NodeFromFileInfo(test.filename, fi)
+			node, err := NodeFromFileInfo(test.filename, fi, false)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/restic/node_windows.go

@@ -81,6 +81,10 @@ func Listxattr(path string) ([]string, error) {
 	return nil, nil
 }
 
+func IsListxattrPermissionError(_ error) bool {
+	return false
+}
+
 // Setxattr associates name and data together as an attribute of path.
 func Setxattr(path, name string, data []byte) error {
 	return nil

internal/restic/node_windows_test.go

@@ -222,7 +222,7 @@ func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpec
 	fi, err := os.Lstat(testPath)
 	test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", testPath))
 
-	nodeFromFileInfo, err := NodeFromFileInfo(testPath, fi)
+	nodeFromFileInfo, err := NodeFromFileInfo(testPath, fi, false)
 	test.OK(t, errors.Wrapf(err, "Could not get NodeFromFileInfo for path: %s", testPath))
 
 	return testPath, nodeFromFileInfo

internal/restic/node_xattr.go

@@ -25,6 +25,14 @@ func Listxattr(path string) ([]string, error) {
 	return l, handleXattrErr(err)
 }
 
+func IsListxattrPermissionError(err error) bool {
+	var xerr *xattr.Error
+	if errors.As(err, &xerr) {
+		return xerr.Op == "xattr.list" && errors.Is(xerr.Err, os.ErrPermission)
+	}
+	return false
+}
+
 // Setxattr associates name and data together as an attribute of path.
 func Setxattr(path, name string, data []byte) error {
 	return handleXattrErr(xattr.LSet(path, name, data))

internal/restic/node_xattr_test.go

@@ -0,0 +1,28 @@
+//go:build darwin || freebsd || linux || solaris
+// +build darwin freebsd linux solaris
+
+package restic
+
+import (
+	"os"
+	"testing"
+
+	"github.com/pkg/xattr"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestIsListxattrPermissionError(t *testing.T) {
+	xerr := &xattr.Error{
+		Op:   "xattr.list",
+		Name: "test",
+		Err:  os.ErrPermission,
+	}
+	err := handleXattrErr(xerr)
+	rtest.Assert(t, err != nil, "missing error")
+	rtest.Assert(t, IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return true for %v", err)
+
+	xerr.Err = os.ErrNotExist
+	err = handleXattrErr(xerr)
+	rtest.Assert(t, err != nil, "missing error")
+	rtest.Assert(t, !IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return false for %v", err)
+}

internal/restic/repository.go

@@ -26,6 +26,7 @@ type Repository interface {
 
 	Index() MasterIndex
 	LoadIndex(context.Context, *progress.Counter) error
+	ClearIndex()
 	SetIndex(MasterIndex) error
 	LookupBlobSize(ID, BlobType) (uint, bool)
 
@@ -102,8 +103,8 @@ type MasterIndex interface {
 	Lookup(BlobHandle) []PackedBlob
 
 	// Each runs fn on all blobs known to the index. When the context is cancelled,
-	// the index iteration return immediately. This blocks any modification of the index.
-	Each(ctx context.Context, fn func(PackedBlob))
+	// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
+	Each(ctx context.Context, fn func(PackedBlob)) error
 	ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs
 
 	Save(ctx context.Context, repo Repository, excludePacks IDSet, extraObsolete IDs, opts MasterIndexSaveOpts) error

internal/restic/snapshot.go

@@ -25,11 +25,31 @@ type Snapshot struct {
 	Tags     []string  `json:"tags,omitempty"`
 	Original *ID       `json:"original,omitempty"`
 
-	ProgramVersion string `json:"program_version,omitempty"`
+	ProgramVersion string           `json:"program_version,omitempty"`
+	Summary        *SnapshotSummary `json:"summary,omitempty"`
 
 	id *ID // plaintext ID, used during restore
 }
 
+type SnapshotSummary struct {
+	BackupStart time.Time `json:"backup_start"`
+	BackupEnd   time.Time `json:"backup_end"`
+
+	// statistics from the backup json output
+	FilesNew            uint   `json:"files_new"`
+	FilesChanged        uint   `json:"files_changed"`
+	FilesUnmodified     uint   `json:"files_unmodified"`
+	DirsNew             uint   `json:"dirs_new"`
+	DirsChanged         uint   `json:"dirs_changed"`
+	DirsUnmodified      uint   `json:"dirs_unmodified"`
+	DataBlobs           int    `json:"data_blobs"`
+	TreeBlobs           int    `json:"tree_blobs"`
+	DataAdded           uint64 `json:"data_added"`
+	DataAddedPacked     uint64 `json:"data_added_packed"`
+	TotalFilesProcessed uint   `json:"total_files_processed"`
+	TotalBytesProcessed uint64 `json:"total_bytes_processed"`
+}
+
 // NewSnapshot returns an initialized snapshot struct for the current user and
 // time.
 func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) {

internal/restic/testing_test.go

@@ -45,7 +45,7 @@ func TestCreateSnapshot(t *testing.T) {
 		t.Fatalf("snapshot has zero tree ID")
 	}
 
-	checker.TestCheckRepo(t, repo)
+	checker.TestCheckRepo(t, repo, false)
 }
 
 func BenchmarkTestCreateSnapshot(t *testing.B) {

internal/restic/tree_test.go

@@ -86,7 +86,7 @@ func TestNodeComparison(t *testing.T) {
 	fi, err := os.Lstat("tree_test.go")
 	rtest.OK(t, err)
 
-	node, err := restic.NodeFromFileInfo("tree_test.go", fi)
+	node, err := restic.NodeFromFileInfo("tree_test.go", fi, false)
 	rtest.OK(t, err)
 
 	n2 := *node
@@ -127,7 +127,7 @@ func TestTreeEqualSerialization(t *testing.T) {
 		for _, fn := range files[:i] {
 			fi, err := os.Lstat(fn)
 			rtest.OK(t, err)
-			node, err := restic.NodeFromFileInfo(fn, fi)
+			node, err := restic.NodeFromFileInfo(fn, fi, false)
 			rtest.OK(t, err)
 
 			rtest.OK(t, tree.Insert(node))

internal/restorer/doc.go

@@ -18,7 +18,7 @@
 //
 // Implementation does not guarantee order in which blobs are written to the
 // target files and, for example, the last blob of a file can be written to the
-// file before any of the preceeding file blobs. It is therefore possible to
+// file before any of the preceding file blobs. It is therefore possible to
 // have gaps in the data written to the target files if restore fails or
 // interrupted by the user.
 package restorer

internal/restorer/restorer_test.go

@@ -858,7 +858,7 @@ func TestRestorerSparseFiles(t *testing.T) {
 	rtest.OK(t, err)
 
 	arch := archiver.New(repo, target, archiver.Options{})
-	sn, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
+	sn, _, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
 		archiver.SnapshotOptions{})
 	rtest.OK(t, err)
 

internal/test/helpers.go

@@ -5,6 +5,7 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -12,8 +13,6 @@ import (
 	"testing"
 
 	"github.com/restic/restic/internal/errors"
-
-	mrand "math/rand"
 )
 
 // Assert fails the test if the condition is false.
@@ -71,7 +70,7 @@ func Equals(tb testing.TB, exp, act interface{}, msgs ...string) {
 func Random(seed, count int) []byte {
 	p := make([]byte, count)
 
-	rnd := mrand.New(mrand.NewSource(int64(seed)))
+	rnd := rand.New(rand.NewSource(int64(seed)))
 
 	for i := 0; i < len(p); i += 8 {
 		val := rnd.Int63()

internal/ui/backup/json.go

@@ -163,7 +163,7 @@ func (b *JSONProgress) ReportTotal(start time.Time, s archiver.ScanStats) {
 }
 
 // Finish prints the finishing messages.
-func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) {
+func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
 	b.print(summaryOutput{
 		MessageType:         "summary",
 		FilesNew:            summary.Files.New,
@@ -175,6 +175,7 @@ func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su
 		DataBlobs:           summary.ItemStats.DataBlobs,
 		TreeBlobs:           summary.ItemStats.TreeBlobs,
 		DataAdded:           summary.ItemStats.DataSize + summary.ItemStats.TreeSize,
+		DataAddedPacked:     summary.ItemStats.DataSizeInRepo + summary.ItemStats.TreeSizeInRepo,
 		TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged,
 		TotalBytesProcessed: summary.ProcessedBytes,
 		TotalDuration:       time.Since(start).Seconds(),
@@ -230,6 +231,7 @@ type summaryOutput struct {
 	DataBlobs           int     `json:"data_blobs"`
 	TreeBlobs           int     `json:"tree_blobs"`
 	DataAdded           uint64  `json:"data_added"`
+	DataAddedPacked     uint64  `json:"data_added_packed"`
 	TotalFilesProcessed uint    `json:"total_files_processed"`
 	TotalBytesProcessed uint64  `json:"total_bytes_processed"`
 	TotalDuration       float64 `json:"total_duration"` // in seconds

internal/ui/backup/progress.go

@@ -17,7 +17,7 @@ type ProgressPrinter interface {
 	ScannerError(item string, err error) error
 	CompleteItem(messageType string, item string, s archiver.ItemStats, d time.Duration)
 	ReportTotal(start time.Time, s archiver.ScanStats)
-	Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool)
+	Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool)
 	Reset()
 
 	P(msg string, args ...interface{})
@@ -28,16 +28,6 @@ type Counter struct {
 	Files, Dirs, Bytes uint64
 }
 
-type Summary struct {
-	Files, Dirs struct {
-		New       uint
-		Changed   uint
-		Unchanged uint
-	}
-	ProcessedBytes uint64
-	archiver.ItemStats
-}
-
 // Progress reports progress for the `backup` command.
 type Progress struct {
 	progress.Updater
@@ -52,7 +42,6 @@ type Progress struct {
 	processed, total Counter
 	errors           uint
 
-	summary Summary
 	printer ProgressPrinter
 }
 
@@ -126,16 +115,6 @@ func (p *Progress) CompleteBlob(bytes uint64) {
 // CompleteItem is the status callback function for the archiver when a
 // file/dir has been saved successfully.
 func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
-	p.mu.Lock()
-	p.summary.ItemStats.Add(s)
-
-	// for the last item "/", current is nil
-	if current != nil {
-		p.summary.ProcessedBytes += current.Size
-	}
-
-	p.mu.Unlock()
-
 	if current == nil {
 		// error occurred, tell the status display to remove the line
 		p.mu.Lock()
@@ -153,21 +132,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
 		switch {
 		case previous == nil:
 			p.printer.CompleteItem("dir new", item, s, d)
-			p.mu.Lock()
-			p.summary.Dirs.New++
-			p.mu.Unlock()
-
 		case previous.Equals(*current):
 			p.printer.CompleteItem("dir unchanged", item, s, d)
-			p.mu.Lock()
-			p.summary.Dirs.Unchanged++
-			p.mu.Unlock()
-
 		default:
 			p.printer.CompleteItem("dir modified", item, s, d)
-			p.mu.Lock()
-			p.summary.Dirs.Changed++
-			p.mu.Unlock()
 		}
 
 	case "file":
@@ -179,21 +147,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
 		switch {
 		case previous == nil:
 			p.printer.CompleteItem("file new", item, s, d)
-			p.mu.Lock()
-			p.summary.Files.New++
-			p.mu.Unlock()
-
 		case previous.Equals(*current):
 			p.printer.CompleteItem("file unchanged", item, s, d)
-			p.mu.Lock()
-			p.summary.Files.Unchanged++
-			p.mu.Unlock()
-
 		default:
 			p.printer.CompleteItem("file modified", item, s, d)
-			p.mu.Lock()
-			p.summary.Files.Changed++
-			p.mu.Unlock()
 		}
 	}
 }
@@ -213,8 +170,8 @@ func (p *Progress) ReportTotal(item string, s archiver.ScanStats) {
 }
 
 // Finish prints the finishing messages.
-func (p *Progress) Finish(snapshotID restic.ID, dryrun bool) {
+func (p *Progress) Finish(snapshotID restic.ID, summary *archiver.Summary, dryrun bool) {
 	// wait for the status update goroutine to shut down
 	p.Updater.Done()
-	p.printer.Finish(snapshotID, p.start, &p.summary, dryrun)
+	p.printer.Finish(snapshotID, p.start, summary, dryrun)
 }

internal/ui/backup/progress_test.go

@@ -33,11 +33,10 @@ func (p *mockPrinter) CompleteItem(messageType string, _ string, _ archiver.Item
 }
 
 func (p *mockPrinter) ReportTotal(_ time.Time, _ archiver.ScanStats) {}
-func (p *mockPrinter) Finish(id restic.ID, _ time.Time, summary *Summary, _ bool) {
+func (p *mockPrinter) Finish(id restic.ID, _ time.Time, _ *archiver.Summary, _ bool) {
 	p.Lock()
 	defer p.Unlock()
 
-	_ = *summary // Should not be nil.
 	p.id = id
 }
 
@@ -64,7 +63,7 @@ func TestProgress(t *testing.T) {
 
 	time.Sleep(10 * time.Millisecond)
 	id := restic.NewRandomID()
-	prog.Finish(id, false)
+	prog.Finish(id, nil, false)
 
 	if !prnt.dirUnchanged {
 		t.Error(`"dir unchanged" event not seen`)

internal/ui/backup/text.go

@@ -126,7 +126,7 @@ func (b *TextProgress) Reset() {
 }
 
 // Finish prints the finishing messages.
-func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *Summary, dryRun bool) {
+func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
 	b.P("\n")
 	b.P("Files:       %5d new, %5d changed, %5d unmodified\n", summary.Files.New, summary.Files.Changed, summary.Files.Unchanged)
 	b.P("Dirs:        %5d new, %5d changed, %5d unmodified\n", summary.Dirs.New, summary.Dirs.Changed, summary.Dirs.Unchanged)

internal/ui/progress/printer.go

@@ -1,5 +1,7 @@
 package progress
 
+import "testing"
+
 // A Printer can can return a new counter or print messages
 // at different log levels.
 // It must be safe to call its methods from concurrent goroutines.
@@ -28,3 +30,36 @@ func (*NoopPrinter) P(_ string, _ ...interface{}) {}
 func (*NoopPrinter) V(_ string, _ ...interface{}) {}
 
 func (*NoopPrinter) VV(_ string, _ ...interface{}) {}
+
+// TestPrinter prints messages during testing
+type TestPrinter struct {
+	t testing.TB
+}
+
+func NewTestPrinter(t testing.TB) *TestPrinter {
+	return &TestPrinter{
+		t: t,
+	}
+}
+
+var _ Printer = (*TestPrinter)(nil)
+
+func (p *TestPrinter) NewCounter(_ string) *Counter {
+	return nil
+}
+
+func (p *TestPrinter) E(msg string, args ...interface{}) {
+	p.t.Logf("error: "+msg, args...)
+}
+
+func (p *TestPrinter) P(msg string, args ...interface{}) {
+	p.t.Logf("print: "+msg, args...)
+}
+
+func (p *TestPrinter) V(msg string, args ...interface{}) {
+	p.t.Logf("verbose: "+msg, args...)
+}
+
+func (p *TestPrinter) VV(msg string, args ...interface{}) {
+	p.t.Logf("verbose2: "+msg, args...)
+}

internal/walker/walker_test.go

@@ -13,7 +13,7 @@ import (
 // TestTree is used to construct a list of trees for testing the walker.
 type TestTree map[string]interface{}
 
-// TestNode is used to test the walker.
+// TestFile is used to test the walker.
 type TestFile struct {
 	Size uint64
 }