Merge pull request #21902 from restic/speedup-test

Cut local test suite execution time in half

cmd/restic/cmd_backup_integration_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/global"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 	"github.com/restic/restic/internal/ui/backup"
@@ -60,6 +61,9 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 
+	// Use auto compression to ensure coverage in the integration tests
+	// All other tests use fastest compression for faster execution
+	env.gopts.Compression = repository.CompressionAuto
 	testSetupBackupData(t, env)
 	opts := BackupOptions{UseFsSnapshot: useFsSnapshot}
 

cmd/restic/cmd_init_integration_test.go

@@ -25,6 +25,7 @@ func testRunInit(t testing.TB, gopts global.Options) {
 
 	// create temporary junk files to verify that restic does not trip over them
 	for _, path := range []string{"index", "snapshots", "keys", "locks", filepath.Join("data", "00")} {
+		rtest.OK(t, os.MkdirAll(filepath.Join(gopts.Repo, path), 0700))
 		rtest.OK(t, os.WriteFile(filepath.Join(gopts.Repo, path, "tmp12345"), []byte("junk file"), 0o600))
 	}
 }

cmd/restic/cmd_mount_integration_test.go

@@ -14,7 +14,6 @@ import (
 
 	systemFuse "github.com/anacrolix/fuse"
 	"github.com/restic/restic/internal/data"
-	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -22,8 +21,8 @@ import (
 )
 
 const (
-	mountWait       = 20
-	mountSleep      = 100 * time.Millisecond
+	mountWait       = 400
+	mountSleep      = 5 * time.Millisecond
 	mountTestSubdir = "snapshots"
 )
 
@@ -274,11 +273,6 @@ func TestMountSameTimestamps(t *testing.T) {
 		t.Skip("Skipping fuse tests")
 	}
 
-	debugEnabled := debug.TestLogToStderr(t)
-	if debugEnabled {
-		defer debug.TestDisableLog(t)
-	}
-
 	env, cleanup := withTestEnvironment(t)
 	// must list snapshots more than once
 	env.gopts.BackendTestHook = nil

cmd/restic/cmd_repair_index_integration_test.go

@@ -11,6 +11,8 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -22,6 +24,7 @@ func testRunRebuildIndex(t testing.TB, gopts global.Options) {
 }
 
 func testRebuildIndex(t *testing.T, backendTestHook global.BackendWrapper) {
+	repository.TestInjectKey(t, restic.TestParseID("b9883c60bed42db51be171ca52f055104b6ea7cfa2bc381c05b2b1f78231280c"), `{"mac":{"k":"maQ4ILA872XnDxHVEno94A==","r":"OptMBABwkgIsMQcHME8cBw=="},"encrypt":"janrR1efN7HyQ8kOZ9zhHixooZ/e+WelH0mT4v9WskQ="}`)
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 
@@ -109,6 +112,7 @@ func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error {
 }
 
 func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("b9883c60bed42db51be171ca52f055104b6ea7cfa2bc381c05b2b1f78231280c"), `{"mac":{"k":"maQ4ILA872XnDxHVEno94A==","r":"OptMBABwkgIsMQcHME8cBw=="},"encrypt":"janrR1efN7HyQ8kOZ9zhHixooZ/e+WelH0mT4v9WskQ="}`)
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 

cmd/restic/cmd_restore_integration_test.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/global"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -329,6 +330,7 @@ func TestRestoreLatest(t *testing.T) {
 }
 
 func TestRestoreWithPermissionFailure(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("18493b1f93ad90b6bce7ed3afa93395a1f90e981f04145c03e5958cafa2ee33b"), `{"mac":{"k":"7ftgSq7jNM2HiGCyY9TYrg==","r":"o+1bB0wApwqoG7oAXOLyDw=="},"encrypt":"i57gVfyYp9HmXjzE0dSZyrkp2FN9LE75uWFjOuWze1M="}`)
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 

cmd/restic/integration_helpers_test.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/all"
+	"github.com/restic/restic/internal/backend/layout"
 	"github.com/restic/restic/internal/backend/retry"
 	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
@@ -192,6 +193,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 	repository.TestUseLowSecurityKDFParameters(t)
 	restic.TestDisableCheckPolynomial(t)
 	retry.TestFastRetries(t)
+	layout.TestDisablePackSubdirs(t)
 
 	tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-")
 	rtest.OK(t, err)
@@ -210,11 +212,12 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 	rtest.OK(t, os.MkdirAll(env.repo, 0700))
 
 	env.gopts = global.Options{
-		Repo:     env.repo,
-		Quiet:    true,
-		CacheDir: env.cache,
-		Password: rtest.TestPassword,
-		Extended: make(options.Options),
+		Repo:        env.repo,
+		Quiet:       true,
+		CacheDir:    env.cache,
+		Password:    rtest.TestPassword,
+		Extended:    make(options.Options),
+		Compression: repository.CompressionFastest,
 
 		// replace this hook with "nil" if listing a filetype more than once is necessary
 		BackendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil },

cmd/restic/integration_test.go

@@ -12,12 +12,14 @@ import (
 	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/global"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 	"github.com/restic/restic/internal/ui/progress"
 )
 
 func TestCheckRestoreNoLock(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("a19acdab068765b022ffb81cb5aac83c5de4bf4fbce0d26e9ade8e636c6ae49f"), `{"mac":{"k":"TbkpCBdNYAvAwb+64r8VGw==","r":"Q5V1CnAvBQREgJAOQD40Bw=="},"encrypt":"SjCkTpms+XOUJR5LSsy2G+uO9ngG7H0L+IVwPV4u70A="}`)
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 

internal/archiver/archiver_test.go

@@ -572,13 +572,7 @@ func nodeFromFile(t testing.TB, localFs fs.FS, filename string) *data.Node {
 
 // sleep sleeps long enough to ensure a timestamp change.
 func sleep() {
-	d := 50 * time.Millisecond
-	if runtime.GOOS == "darwin" {
-		// On older Darwin instances, the file system only supports one second
-		// granularity.
-		d = 1500 * time.Millisecond
-	}
-	time.Sleep(d)
+	time.Sleep(5 * time.Millisecond)
 }
 
 func TestFileChanged(t *testing.T) {
@@ -754,12 +748,12 @@ func TestArchiverSaveDir(t *testing.T) {
 	}{
 		{
 			src: TestDir{
-				"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
+				"targetfile": TestFile{Content: string(rtest.Random(888, 20*1024+5000))},
 			},
 			target: ".",
 			want: TestDir{
 				"targetdir": TestDir{
-					"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
+					"targetfile": TestFile{Content: string(rtest.Random(888, 20*1024+5000))},
 				},
 			},
 		},
@@ -769,8 +763,8 @@ func TestArchiverSaveDir(t *testing.T) {
 					"foo":        TestFile{Content: "foo"},
 					"emptyfile":  TestFile{Content: ""},
 					"bar":        TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"},
-					"largefile":  TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
-					"largerfile": TestFile{Content: string(rtest.Random(234, 5*1024*1024+5000))},
+					"largefile":  TestFile{Content: string(rtest.Random(888, 1*1024*1024+5000))},
+					"largerfile": TestFile{Content: string(rtest.Random(234, 3*1024*1024+5000))},
 				},
 			},
 			target: "targetdir",

internal/backend/cache/file_test.go

@@ -22,7 +22,7 @@ import (
 func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} {
 	ids := make(map[string]struct{})
 	for i := 0; i < random.Intn(15)+10; i++ {
-		buf := rtest.Random(random.Int(), 1<<19)
+		buf := rtest.Random(random.Int(), 1<<15)
 		id := restic.Hash(buf)
 		h := backend.Handle{Type: tpe, Name: id.String()}
 

internal/backend/layout/layout_default.go

@@ -6,6 +6,10 @@ import (
 	"github.com/restic/restic/internal/backend"
 )
 
+// disablePackSubdirs is used to disable the creation of pack subdirectories.
+// Only used for testing.
+var disablePackSubdirs = false
+
 // DefaultLayout implements the default layout for local and sftp backends, as
 // described in the Design document. The `data` directory has one level of
 // subdirs, two characters each (taken from the first two characters of the
@@ -66,10 +70,12 @@ func (l *DefaultLayout) Paths() (dirs []string) {
 		dirs = append(dirs, l.join(l.path, p))
 	}
 
-	// also add subdirs
-	for i := 0; i < 256; i++ {
-		subdir := hex.EncodeToString([]byte{byte(i)})
-		dirs = append(dirs, l.join(l.path, defaultLayoutPaths[backend.PackFile], subdir))
+	if !disablePackSubdirs {
+		// also add subdirs
+		for i := 0; i < 256; i++ {
+			subdir := hex.EncodeToString([]byte{byte(i)})
+			dirs = append(dirs, l.join(l.path, defaultLayoutPaths[backend.PackFile], subdir))
+		}
 	}
 
 	return dirs

internal/backend/layout/testing.go

@@ -0,0 +1,9 @@
+package layout
+
+import (
+	"testing"
+)
+
+func TestDisablePackSubdirs(t testing.TB) {
+	disablePackSubdirs = true
+}

internal/backend/rclone/backend_test.go

@@ -34,6 +34,7 @@ func findRclone(t testing.TB) {
 }
 
 func TestBackendRclone(t *testing.T) {
+	t.Parallel()
 	defer func() {
 		if t.Skipped() {
 			rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone")

internal/backend/rclone/internal_test.go

@@ -12,6 +12,7 @@ import (
 
 // restic should detect rclone exiting.
 func TestRcloneExit(t *testing.T) {
+	t.Parallel()
 	dir := rtest.TempDir(t)
 	cfg := NewConfig()
 	cfg.Remote = dir

internal/backend/test/tests.go

@@ -143,7 +143,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
 	test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize non-existing blob: %v", err)
 	test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize non-existing blob: %v", err)
 
-	length := random.Intn(1<<24) + 2000
+	length := random.Intn(1<<20) + 2000
 
 	data := test.Random(23, length)
 	id := restic.Hash(data)
@@ -426,10 +426,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
 		}
 	})
 
-	t.Run("Timeout", func(t *testing.T) {
-		// rather large timeout, let's try to get at least one item
-		timeout := time.Second
-
+	testTimeout := func(timeout time.Duration) error {
 		ctxTimeout, cancel := context.WithTimeout(context.TODO(), timeout)
 		defer cancel()
 
@@ -449,11 +446,28 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
 		})
 
 		if !errors.Is(err, context.DeadlineExceeded) {
-			t.Fatalf("expected error not found, want %#v, got %#v", context.DeadlineExceeded, err)
+			return errors.Errorf("expected error not found, want %#v, got %#v", context.DeadlineExceeded, err)
 		}
 
 		if i > 2 {
-			t.Fatalf("wrong number of files returned by List, want <= 2, got %v", i)
+			return errors.Errorf("wrong number of files returned by List, want <= 2, got %v", i)
+		}
+
+		return nil
+	}
+
+	t.Run("Timeout", func(t *testing.T) {
+		// try short timeouts first to speed up tests for fast backends
+		var err error
+		for _, timeout := range []time.Duration{10 * time.Millisecond, 100 * time.Millisecond, 1 * time.Second} {
+			err = testTimeout(timeout)
+			if err == nil {
+				break
+			}
+		}
+		// fails if last attempt also did not succeed
+		if err != nil {
+			t.Fatal(err)
 		}
 	})
 

internal/checker/checker_test.go

@@ -12,7 +12,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/data"
@@ -73,6 +72,7 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) {
 }
 
 func TestCheckRepo(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, _ := repository.TestFromFixture(t, checkerTestData)
 
 	chkr := checker.New(repo, false)
@@ -90,6 +90,7 @@ func TestCheckRepo(t *testing.T) {
 }
 
 func TestMissingPack(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, be := repository.TestFromFixture(t, checkerTestData)
 
 	packID := restic.TestParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6")
@@ -115,6 +116,7 @@ func TestMissingPack(t *testing.T) {
 }
 
 func TestUnreferencedPack(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, be := repository.TestFromFixture(t, checkerTestData)
 
 	// index 3f1a only references pack 60e0
@@ -142,6 +144,7 @@ func TestUnreferencedPack(t *testing.T) {
 }
 
 func TestUnreferencedBlobs(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, be := repository.TestFromFixture(t, checkerTestData)
 
 	snapshotID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
@@ -176,6 +179,7 @@ func TestUnreferencedBlobs(t *testing.T) {
 }
 
 func TestModifiedIndex(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, be := repository.TestFromFixture(t, checkerTestData)
 
 	done := make(chan struct{})
@@ -215,6 +219,7 @@ func TestModifiedIndex(t *testing.T) {
 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 
 func TestDuplicatePacksInIndex(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("b9883c60bed42db51be171ca52f055104b6ea7cfa2bc381c05b2b1f78231280c"), `{"mac":{"k":"maQ4ILA872XnDxHVEno94A==","r":"OptMBABwkgIsMQcHME8cBw=="},"encrypt":"janrR1efN7HyQ8kOZ9zhHixooZ/e+WelH0mT4v9WskQ="}`)
 	repo, _ := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
 
 	chkr := checker.New(repo, false)
@@ -310,7 +315,7 @@ func (b *errorOnceBackend) Load(ctx context.Context, h backend.Handle, length in
 
 func TestCheckerModifiedData(t *testing.T) {
 	repo, _, be := repository.TestRepositoryWithVersion(t, 0)
-	sn := archiver.TestSnapshot(t, repo, ".", nil)
+	sn := data.TestCreateSnapshot(t, repo, time.Unix(1470492820, 207401672), 2)
 	t.Logf("archived as %v", sn.ID().Str())
 
 	errBe := &errorBackend{Backend: be}
@@ -414,6 +419,7 @@ func (r *loadTreesOnceRepository) LoadBlob(ctx context.Context, bh restic.BlobHa
 }
 
 func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, _ := repository.TestFromFixture(t, checkerTestData)
 	checkRepo := &loadTreesOnceRepository{
 		Repository:  repo,
@@ -563,6 +569,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
 }
 
 func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, _ := repository.TestFromFixture(t, checkerTestData)
 
 	chkr := checker.New(repo, false)

internal/data/snapshot_policy_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 	"time"
 
@@ -275,11 +276,12 @@ func TestApplyPolicy(t *testing.T) {
 
 			cmpOpts := cmpopts.IgnoreUnexported(data.Snapshot{})
 
-			if !cmp.Equal(want.Keep, keep, cmpOpts) {
+			// reflect.DeepEqual is faster than cmp.Equal
+			if !reflect.DeepEqual(want.Keep, keep) {
 				t.Error(cmp.Diff(want.Keep, keep, cmpOpts))
 			}
 
-			if !cmp.Equal(want.Reasons, reasons, cmpOpts) {
+			if !reflect.DeepEqual(want.Reasons, reasons) {
 				t.Error(cmp.Diff(want.Reasons, reasons, cmpOpts))
 			}
 		})

internal/fs/fs_reader_test.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"slices"
 	"sort"
 	"strings"
 	"testing"
@@ -23,7 +24,8 @@ func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte
 	test.OK(t, err)
 	test.OK(t, f.Close())
 
-	if !cmp.Equal(want, buf) {
+	// slices.Equal is much faster than cmp.Equal
+	if !slices.Equal(want, buf) {
 		t.Error(cmp.Diff(want, buf))
 	}
 }

internal/fs/node_windows_test.go

@@ -652,6 +652,7 @@ func TestPrepareVolumeName(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
 			isEASupported, err := checkAndStoreEASupport(tc.path)
 			test.OK(t, err)
 			test.Equals(t, tc.expectedEASupported, isEASupported)

internal/repository/checker_test.go

@@ -8,10 +8,11 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/klauspost/compress/zstd"
-	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
@@ -33,6 +34,7 @@ func testWrapCheckPack(ctx context.Context, t *testing.T, repo *Repository,
 
 // TestGapInBlobs creates a gap in the blob list by omitting the first entry before passing it to checkPack
 func TestGapInBlobs(t *testing.T) {
+	TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, _ := TestFromFixture(t, checkerTestData)
 
 	err := repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)
@@ -153,7 +155,7 @@ func setupChecker(t *testing.T, wrap func(backend.Backend) backend.Backend) *Che
 	t.Helper()
 	// Write a snapshot into a fresh in-memory repository.
 	repo, be := TestRepositoryWithBackend(t, nil, 0, Options{})
-	_ = archiver.TestSnapshot(t, repo, ".", nil)
+	data.TestCreateSnapshot(t, repo, time.Unix(1470492820, 207401672), 2)
 
 	// Re-open the same backend (now containing real pack files) through
 	// the corruption wrapper so the checker reads corrupted data.

internal/repository/crypto/kdf_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func TestCalibrate(t *testing.T) {
-	params, err := Calibrate(100*time.Millisecond, 50)
+	params, err := Calibrate(25*time.Millisecond, 50)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/repository/index/index_internal_test.go

@@ -13,11 +13,18 @@ func TestIndexOversized(t *testing.T) {
 
 	// Add blobs up to indexMaxBlobs + pack.MaxHeaderEntries - 1
 	packID := idx.addToPacks(restic.NewRandomID())
+	id := restic.NewRandomID()
 	for i := uint(0); i < indexMaxBlobs+pack.MaxHeaderEntries-1; i++ {
+		// Directly modify ID to avoid benchmarking NewRandomID
+		id[0] = byte(i)
+		id[1] = byte(i >> 8)
+		id[2] = byte(i >> 16)
+		id[3] = byte(i >> 24)
+
 		idx.store(packID, pack.Blob{
 			BlobHandle: restic.BlobHandle{
 				Type: restic.DataBlob,
-				ID:   restic.NewRandomID(),
+				ID:   id,
 			},
 			Length: 100,
 			Offset: uint(i) * 100,

internal/repository/index/index_parallel_test.go

@@ -2,26 +2,44 @@ package index_test
 
 import (
 	"context"
-	"path/filepath"
 	"testing"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/repository/crypto"
 	"github.com/restic/restic/internal/repository/index"
+	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
 
-var repoFixture = filepath.Join("..", "testdata", "test-repo.tar.gz")
-
 func TestRepositoryForAllIndexes(t *testing.T) {
-	repo, _ := repository.TestFromFixture(t, repoFixture)
+	originalFull := index.Full
+	defer func() {
+		index.Full = originalFull
+	}()
+	index.Full = func(*index.Index) bool { return true }
+
+	repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion)
+
+	mi := index.NewMasterIndex()
+	for range 3 {
+		packID := restic.NewRandomID()
+		blob := pack.Blob{
+			BlobHandle: restic.NewRandomBlobHandle(),
+			Length:     uint(crypto.CiphertextLength(10)),
+			Offset:     0,
+		}
+		rtest.OK(t, mi.StorePack(context.TODO(), packID, pack.Blobs{blob}, unpacked))
+		rtest.OK(t, mi.Flush(context.TODO(), unpacked))
+	}
 
 	expectedIndexIDs := restic.NewIDSet()
 	rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
 		expectedIndexIDs.Insert(id)
 		return nil
 	}))
+	rtest.Assert(t, len(expectedIndexIDs) > 1, "test repo should have multiple indexes")
 
 	// check that all expected indexes are loaded without errors
 	indexIDs := restic.NewIDSet()

internal/repository/index/master_index_test.go

@@ -410,7 +410,7 @@ func BenchmarkMasterIndexGC(b *testing.B) {
 
 var (
 	snapshotTime = time.Unix(1470492820, 207401672)
-	depth        = 3
+	depth        = 2
 )
 
 func createFilledRepo(t testing.TB, snapshots int, version uint) (*repository.Repository, restic.Unpacked[restic.FileType]) {

internal/repository/key.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"os/user"
+	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/errors"
@@ -47,6 +48,9 @@ type Key struct {
 // calibrated on the first run of AddKey().
 var params *crypto.Params
 
+// testKeyInjection is used to speed up tests by skipping the key decryption step.
+var testKeyInjection = sync.Map{}
+
 const (
 	// KDFTimeout specifies the maximum runtime for the KDF.
 	KDFTimeout = 500 * time.Millisecond
@@ -63,6 +67,14 @@ func createMasterKey(ctx context.Context, s *Repository, password string) (*Key,
 
 // openKey tries do decrypt the key specified by name with the given password.
 func openKey(ctx context.Context, s *Repository, id restic.ID, password string) (*Key, error) {
+	if key, ok := testKeyInjection.Load(id); ok {
+		return &Key{
+			master: key.(*crypto.Key),
+			user:   key.(*crypto.Key), // not correct but good enough for testing
+			id:     id,
+		}, nil
+	}
+
 	k, err := LoadKey(ctx, s, id)
 	if err != nil {
 		debug.Log("LoadKey(%v) returned error %v", id.String(), err)

internal/repository/prune_test.go

@@ -120,12 +120,13 @@ func TestPrune(t *testing.T) {
 6.) The result should be less packfiles than before
 */
 func TestPruneSmall(t *testing.T) {
+	t.Parallel()
 	seed := time.Now().UnixNano()
 	random := rand.New(rand.NewSource(seed))
 	t.Logf("rand initialized with seed %d", seed)
 
 	be := repository.TestBackend(t)
-	repo, _ := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{PackSize: repository.MinPackSize})
+	repo, _ := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{PackSize: repository.MinPackSize, Compression: repository.CompressionOff})
 
 	const blobSize = 1000 * 1000
 	const numBlobsCreated = 55

internal/repository/repository.go

@@ -919,13 +919,10 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch
 		return err
 	}
 
-	cfg, err := restic.CreateConfig(version)
+	cfg, err := restic.CreateConfig(version, chunkerPolynomial)
 	if err != nil {
 		return err
 	}
-	if chunkerPolynomial != nil {
-		cfg.ChunkerPolynomial = *chunkerPolynomial
-	}
 
 	return r.init(ctx, password, cfg)
 }

internal/repository/repository_internal_test.go

@@ -205,19 +205,17 @@ func testStreamPack(t *testing.T, version uint) {
 	key := testKey(t)
 
 	blobSizes := []int{
-		5522811,
+		1522811,
 		10,
 		5231,
 		18812,
 		123123,
-		13522811,
 		12301,
 		892242,
 		28616,
 		13351,
 		252287,
 		188883,
-		3522811,
 		18883,
 	}
 

internal/repository/repository_test.go

@@ -28,8 +28,6 @@ import (
 
 var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
 
-var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
-
 func TestSave(t *testing.T) {
 	repository.TestAllVersions(t, testSavePassID)
 	repository.TestAllVersions(t, testSaveCalculateID)
@@ -45,6 +43,7 @@ func testSaveCalculateID(t *testing.T, version uint) {
 
 func testSave(t *testing.T, version uint, calculateID bool) {
 	repo, _, _ := repository.TestRepositoryWithVersion(t, version)
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
@@ -119,6 +118,7 @@ func testSavePackMerging(t *testing.T, targetPercentage int, expectedPacks int)
 		// minimum pack size to speed up test
 		PackSize: repository.MinPackSize,
 	})
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	var ids restic.IDs
 	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
@@ -162,6 +162,7 @@ func benchmarkSaveAndEncrypt(t *testing.B, version uint) {
 	size := 4 << 20 // 4MiB
 
 	data := make([]byte, size)
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 	_, err := io.ReadFull(rnd, data)
 	rtest.OK(t, err)
 
@@ -188,6 +189,7 @@ func testLoadBlob(t *testing.T, version uint) {
 	repo, _, _ := repository.TestRepositoryWithVersion(t, version)
 	length := 1000000
 	buf := crypto.NewBlobBuffer(length)
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 	_, err := io.ReadFull(rnd, buf)
 	rtest.OK(t, err)
 
@@ -245,6 +247,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) {
 	repo, _, _ := repository.TestRepositoryWithVersion(b, version)
 	length := 1000000
 	buf := crypto.NewBlobBuffer(length)
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 	_, err := io.ReadFull(rnd, buf)
 	rtest.OK(b, err)
 
@@ -286,6 +289,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
 	repo, _, _ := repository.TestRepositoryWithVersion(b, version)
 	length := 1000000
 	buf := crypto.NewBlobBuffer(length)
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 	_, err := io.ReadFull(rnd, buf)
 	rtest.OK(b, err)
 
@@ -319,6 +323,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
 var repoFixture = filepath.Join("testdata", "test-repo.tar.gz")
 
 func TestRepositoryLoadIndex(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repo, _ := repository.TestFromFixture(t, repoFixture)
 
 	rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory))
@@ -372,6 +377,7 @@ func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length
 }
 
 func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
+	repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`)
 	repodir := rtest.Env(t, repoFixture)
 
 	be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2}, t.Logf)
@@ -383,9 +389,10 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
 
 // saveRandomDataBlobs generates random data blobs and saves them to the repository.
 func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) {
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 	rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
 		for i := 0; i < num; i++ {
-			size := rand.Int() % sizeMax
+			size := rnd.Int() % sizeMax
 
 			buf := make([]byte, size)
 			_, err := io.ReadFull(rnd, buf)
@@ -405,8 +412,6 @@ func TestRepositoryIncrementalIndex(t *testing.T) {
 func testRepositoryIncrementalIndex(t *testing.T, version uint) {
 	repo, _, _ := repository.TestRepositoryWithVersion(t, version)
 
-	index.Full = func(*index.Index) bool { return true }
-
 	// add a few rounds of packs
 	for j := 0; j < 5; j++ {
 		// add some packs and write index
@@ -438,7 +443,6 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
 			t.Errorf("pack %v listed in %d indexes\n", packID, len(ids))
 		}
 	}
-
 }
 
 func TestInvalidCompression(t *testing.T) {

internal/repository/testing.go

@@ -2,6 +2,7 @@ package repository
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"sync"
@@ -54,6 +55,10 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint, o
 	if be == nil {
 		be = TestBackend(t)
 	}
+	// Speed up tests by default
+	if opts.Compression == CompressionAuto {
+		opts.Compression = CompressionFastest
+	}
 
 	repo, err := New(be, opts)
 	if err != nil {
@@ -126,7 +131,7 @@ func TestOpenLocal(t testing.TB, dir string) (*Repository, backend.Backend) {
 }
 
 func TestOpenBackend(t testing.TB, be backend.Backend) *Repository {
-	repo, err := New(be, Options{})
+	repo, err := New(be, Options{Compression: CompressionFastest})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -143,6 +148,7 @@ type VersionedTest func(t *testing.T, version uint)
 func TestAllVersions(t *testing.T, test VersionedTest) {
 	for version := restic.MinRepoVersion; version <= restic.MaxRepoVersion; version++ {
 		t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) {
+			t.Parallel()
 			test(t, uint(version))
 		})
 	}
@@ -189,3 +195,12 @@ func TestCheckRepo(t testing.TB, repo *Repository) {
 		t.Error(err)
 	}
 }
+
+func TestInjectKey(t testing.TB, keyID restic.ID, key string) {
+	var k crypto.Key
+	err := json.Unmarshal([]byte(key), &k)
+	if err != nil {
+		t.Fatal(err)
+	}
+	testKeyInjection.Store(keyID, &k)
+}

internal/restic/config.go

@@ -28,15 +28,19 @@ const StableRepoVersion = 2
 
 // CreateConfig creates a config file with a randomly selected polynomial and
 // ID.
-func CreateConfig(version uint) (Config, error) {
+func CreateConfig(version uint, pol *chunker.Pol) (Config, error) {
 	var (
 		err error
 		cfg Config
 	)
 
-	cfg.ChunkerPolynomial, err = chunker.RandomPolynomial()
-	if err != nil {
-		return Config{}, errors.Wrap(err, "chunker.RandomPolynomial")
+	if pol == nil {
+		cfg.ChunkerPolynomial, err = chunker.RandomPolynomial()
+		if err != nil {
+			return Config{}, errors.Wrap(err, "chunker.RandomPolynomial")
+		}
+	} else {
+		cfg.ChunkerPolynomial = *pol
 	}
 
 	cfg.ID = NewRandomID().String()

internal/restic/config_test.go

@@ -43,7 +43,7 @@ func TestConfig(t *testing.T) {
 		return restic.ID{}, nil
 	}
 
-	cfg1, err := restic.CreateConfig(restic.MaxRepoVersion)
+	cfg1, err := restic.CreateConfig(restic.MaxRepoVersion, nil)
 	rtest.OK(t, err)
 
 	err = restic.SaveConfig(context.TODO(), saver{save}, cfg1)

internal/restorer/filerestorer_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"slices"
+	"strings"
 	"testing"
 
 	"github.com/restic/restic/internal/errors"
@@ -107,9 +108,9 @@ func newTestRepo(content []TestFile) *TestRepo {
 	filesPathToContent := make(map[string]string)
 
 	for _, file := range content {
-		var content string
+		content := strings.Builder{}
 		for _, blob := range file.blobs {
-			content += blob.data
+			content.WriteString(blob.data)
 
 			// get the pack, create as necessary
 			var pack Pack
@@ -134,7 +135,7 @@ func newTestRepo(content []TestFile) *TestRepo {
 
 			packs[blob.pack] = pack
 		}
-		filesPathToContent[file.name] = content
+		filesPathToContent[file.name] = content.String()
 	}
 
 	blobs := make(map[restic.ID][]restic.PackBlob)

internal/test/helpers.go

@@ -195,8 +195,12 @@ func resetReadOnly(t testing.TB, dir string) {
 // afterwards uses os.RemoveAll() to remove the path.
 func RemoveAll(t testing.TB, path string) {
 	t.Helper()
-	resetReadOnly(t, path)
-	err := os.RemoveAll(path)
+	var err error
+	err = os.RemoveAll(path)
+	if err != nil {
+		resetReadOnly(t, path)
+		err = os.RemoveAll(path)
+	}
 	if errors.Is(err, os.ErrNotExist) {
 		err = nil
 	}