diff --git a/cmd/restic/cmd_backup_integration_test.go b/cmd/restic/cmd_backup_integration_test.go index be35da909..cfca46ce1 100644 --- a/cmd/restic/cmd_backup_integration_test.go +++ b/cmd/restic/cmd_backup_integration_test.go @@ -16,6 +16,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/global" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/backup" @@ -60,6 +61,9 @@ func testBackup(t *testing.T, useFsSnapshot bool) { env, cleanup := withTestEnvironment(t) defer cleanup() + // Use auto compression to ensure coverage in the integration tests + // All other tests use fastest compression for faster execution + env.gopts.Compression = repository.CompressionAuto testSetupBackupData(t, env) opts := BackupOptions{UseFsSnapshot: useFsSnapshot} diff --git a/cmd/restic/cmd_init_integration_test.go b/cmd/restic/cmd_init_integration_test.go index d2e96a613..2bfd9c7fb 100644 --- a/cmd/restic/cmd_init_integration_test.go +++ b/cmd/restic/cmd_init_integration_test.go @@ -25,6 +25,7 @@ func testRunInit(t testing.TB, gopts global.Options) { // create temporary junk files to verify that restic does not trip over them for _, path := range []string{"index", "snapshots", "keys", "locks", filepath.Join("data", "00")} { + rtest.OK(t, os.MkdirAll(filepath.Join(gopts.Repo, path), 0700)) rtest.OK(t, os.WriteFile(filepath.Join(gopts.Repo, path, "tmp12345"), []byte("junk file"), 0o600)) } } diff --git a/cmd/restic/cmd_mount_integration_test.go b/cmd/restic/cmd_mount_integration_test.go index 6aa3b8650..5aff2ef92 100644 --- a/cmd/restic/cmd_mount_integration_test.go +++ b/cmd/restic/cmd_mount_integration_test.go @@ -14,7 +14,6 @@ import ( systemFuse "github.com/anacrolix/fuse" "github.com/restic/restic/internal/data" - "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/global" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -22,8 +21,8 @@ import ( ) const ( - mountWait = 20 - mountSleep = 100 * time.Millisecond + mountWait = 400 + mountSleep = 5 * time.Millisecond mountTestSubdir = "snapshots" ) @@ -274,11 +273,6 @@ func TestMountSameTimestamps(t *testing.T) { t.Skip("Skipping fuse tests") } - debugEnabled := debug.TestLogToStderr(t) - if debugEnabled { - defer debug.TestDisableLog(t) - } - env, cleanup := withTestEnvironment(t) // must list snapshots more than once env.gopts.BackendTestHook = nil diff --git a/cmd/restic/cmd_repair_index_integration_test.go b/cmd/restic/cmd_repair_index_integration_test.go index 1b10f88fd..52dc47315 100644 --- a/cmd/restic/cmd_repair_index_integration_test.go +++ b/cmd/restic/cmd_repair_index_integration_test.go @@ -11,6 +11,8 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/global" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -22,6 +24,7 @@ func testRunRebuildIndex(t testing.TB, gopts global.Options) { } func testRebuildIndex(t *testing.T, backendTestHook global.BackendWrapper) { + repository.TestInjectKey(t, restic.TestParseID("b9883c60bed42db51be171ca52f055104b6ea7cfa2bc381c05b2b1f78231280c"), `{"mac":{"k":"maQ4ILA872XnDxHVEno94A==","r":"OptMBABwkgIsMQcHME8cBw=="},"encrypt":"janrR1efN7HyQ8kOZ9zhHixooZ/e+WelH0mT4v9WskQ="}`) env, cleanup := withTestEnvironment(t) defer cleanup() @@ -109,6 +112,7 @@ func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error { } func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("b9883c60bed42db51be171ca52f055104b6ea7cfa2bc381c05b2b1f78231280c"), `{"mac":{"k":"maQ4ILA872XnDxHVEno94A==","r":"OptMBABwkgIsMQcHME8cBw=="},"encrypt":"janrR1efN7HyQ8kOZ9zhHixooZ/e+WelH0mT4v9WskQ="}`) env, cleanup := withTestEnvironment(t) defer cleanup() diff --git a/cmd/restic/cmd_restore_integration_test.go b/cmd/restic/cmd_restore_integration_test.go index 47d611d8b..303b4e7ca 100644 --- a/cmd/restic/cmd_restore_integration_test.go +++ b/cmd/restic/cmd_restore_integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/global" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -329,6 +330,7 @@ func TestRestoreLatest(t *testing.T) { } func TestRestoreWithPermissionFailure(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("18493b1f93ad90b6bce7ed3afa93395a1f90e981f04145c03e5958cafa2ee33b"), `{"mac":{"k":"7ftgSq7jNM2HiGCyY9TYrg==","r":"o+1bB0wApwqoG7oAXOLyDw=="},"encrypt":"i57gVfyYp9HmXjzE0dSZyrkp2FN9LE75uWFjOuWze1M="}`) env, cleanup := withTestEnvironment(t) defer cleanup() diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 614fa916f..c9b287415 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/all" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" @@ -192,6 +193,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { repository.TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) retry.TestFastRetries(t) + layout.TestDisablePackSubdirs(t) tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-") rtest.OK(t, err) @@ -210,11 +212,12 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { rtest.OK(t, os.MkdirAll(env.repo, 0700)) env.gopts = global.Options{ - Repo: env.repo, - Quiet: true, - CacheDir: env.cache, - Password: rtest.TestPassword, - Extended: make(options.Options), + Repo: env.repo, + Quiet: true, + CacheDir: env.cache, + Password: rtest.TestPassword, + Extended: make(options.Options), + Compression: repository.CompressionFastest, // replace this hook with "nil" if listing a filetype more than once is necessary BackendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil }, diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 3bda0dc7d..f9a066ba3 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -12,12 +12,14 @@ import ( "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/global" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/progress" ) func TestCheckRestoreNoLock(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("a19acdab068765b022ffb81cb5aac83c5de4bf4fbce0d26e9ade8e636c6ae49f"), `{"mac":{"k":"TbkpCBdNYAvAwb+64r8VGw==","r":"Q5V1CnAvBQREgJAOQD40Bw=="},"encrypt":"SjCkTpms+XOUJR5LSsy2G+uO9ngG7H0L+IVwPV4u70A="}`) env, cleanup := withTestEnvironment(t) defer cleanup() diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 7eba30e66..6c238263e 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -572,13 +572,7 @@ func nodeFromFile(t testing.TB, localFs fs.FS, filename string) *data.Node { // sleep sleeps long enough to ensure a timestamp change. func sleep() { - d := 50 * time.Millisecond - if runtime.GOOS == "darwin" { - // On older Darwin instances, the file system only supports one second - // granularity. - d = 1500 * time.Millisecond - } - time.Sleep(d) + time.Sleep(5 * time.Millisecond) } func TestFileChanged(t *testing.T) { @@ -754,12 +748,12 @@ func TestArchiverSaveDir(t *testing.T) { }{ { src: TestDir{ - "targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))}, + "targetfile": TestFile{Content: string(rtest.Random(888, 20*1024+5000))}, }, target: ".", want: TestDir{ "targetdir": TestDir{ - "targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))}, + "targetfile": TestFile{Content: string(rtest.Random(888, 20*1024+5000))}, }, }, }, @@ -769,8 +763,8 @@ func TestArchiverSaveDir(t *testing.T) { "foo": TestFile{Content: "foo"}, "emptyfile": TestFile{Content: ""}, "bar": TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}, - "largefile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))}, - "largerfile": TestFile{Content: string(rtest.Random(234, 5*1024*1024+5000))}, + "largefile": TestFile{Content: string(rtest.Random(888, 1*1024*1024+5000))}, + "largerfile": TestFile{Content: string(rtest.Random(234, 3*1024*1024+5000))}, }, }, target: "targetdir", diff --git a/internal/backend/cache/file_test.go b/internal/backend/cache/file_test.go index 949bbcd8c..d26c8cc6a 100644 --- a/internal/backend/cache/file_test.go +++ b/internal/backend/cache/file_test.go @@ -22,7 +22,7 @@ import ( func generateRandomFiles(t testing.TB, random *rand.Rand, tpe backend.FileType, c *Cache) map[string]struct{} { ids := make(map[string]struct{}) for i := 0; i < random.Intn(15)+10; i++ { - buf := rtest.Random(random.Int(), 1<<19) + buf := rtest.Random(random.Int(), 1<<15) id := restic.Hash(buf) h := backend.Handle{Type: tpe, Name: id.String()} diff --git a/internal/backend/layout/layout_default.go b/internal/backend/layout/layout_default.go index d2c4634d3..6566da5a7 100644 --- a/internal/backend/layout/layout_default.go +++ b/internal/backend/layout/layout_default.go @@ -6,6 +6,10 @@ import ( "github.com/restic/restic/internal/backend" ) +// disablePackSubdirs is used to disable the creation of pack subdirectories. +// Only used for testing. +var disablePackSubdirs = false + // DefaultLayout implements the default layout for local and sftp backends, as // described in the Design document. The `data` directory has one level of // subdirs, two characters each (taken from the first two characters of the @@ -66,10 +70,12 @@ func (l *DefaultLayout) Paths() (dirs []string) { dirs = append(dirs, l.join(l.path, p)) } - // also add subdirs - for i := 0; i < 256; i++ { - subdir := hex.EncodeToString([]byte{byte(i)}) - dirs = append(dirs, l.join(l.path, defaultLayoutPaths[backend.PackFile], subdir)) + if !disablePackSubdirs { + // also add subdirs + for i := 0; i < 256; i++ { + subdir := hex.EncodeToString([]byte{byte(i)}) + dirs = append(dirs, l.join(l.path, defaultLayoutPaths[backend.PackFile], subdir)) + } } return dirs diff --git a/internal/backend/layout/testing.go b/internal/backend/layout/testing.go new file mode 100644 index 000000000..c69932776 --- /dev/null +++ b/internal/backend/layout/testing.go @@ -0,0 +1,9 @@ +package layout + +import ( + "testing" +) + +func TestDisablePackSubdirs(t testing.TB) { + disablePackSubdirs = true +} diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go index 742031585..bde6ac024 100644 --- a/internal/backend/rclone/backend_test.go +++ b/internal/backend/rclone/backend_test.go @@ -34,6 +34,7 @@ func findRclone(t testing.TB) { } func TestBackendRclone(t *testing.T) { + t.Parallel() defer func() { if t.Skipped() { rtest.SkipDisallowed(t, "restic/backend/rclone.TestBackendRclone") diff --git a/internal/backend/rclone/internal_test.go b/internal/backend/rclone/internal_test.go index 8da8dcd6f..f6b7336fb 100644 --- a/internal/backend/rclone/internal_test.go +++ b/internal/backend/rclone/internal_test.go @@ -12,6 +12,7 @@ import ( // restic should detect rclone exiting. func TestRcloneExit(t *testing.T) { + t.Parallel() dir := rtest.TempDir(t) cfg := NewConfig() cfg.Remote = dir diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 00c678942..4999bddd9 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -143,7 +143,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) { test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize non-existing blob: %v", err) test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize non-existing blob: %v", err) - length := random.Intn(1<<24) + 2000 + length := random.Intn(1<<20) + 2000 data := test.Random(23, length) id := restic.Hash(data) @@ -426,10 +426,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { } }) - t.Run("Timeout", func(t *testing.T) { - // rather large timeout, let's try to get at least one item - timeout := time.Second - + testTimeout := func(timeout time.Duration) error { ctxTimeout, cancel := context.WithTimeout(context.TODO(), timeout) defer cancel() @@ -449,11 +446,28 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { }) if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("expected error not found, want %#v, got %#v", context.DeadlineExceeded, err) + return errors.Errorf("expected error not found, want %#v, got %#v", context.DeadlineExceeded, err) } if i > 2 { - t.Fatalf("wrong number of files returned by List, want <= 2, got %v", i) + return errors.Errorf("wrong number of files returned by List, want <= 2, got %v", i) + } + + return nil + } + + t.Run("Timeout", func(t *testing.T) { + // try short timeouts first to speed up tests for fast backends + var err error + for _, timeout := range []time.Duration{10 * time.Millisecond, 100 * time.Millisecond, 1 * time.Second} { + err = testTimeout(timeout) + if err == nil { + break + } + } + // fails if last attempt also did not succeed + if err != nil { + t.Fatal(err) } }) diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go index 0de1c24cf..bdc2f9f77 100644 --- a/internal/checker/checker_test.go +++ b/internal/checker/checker_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/data" @@ -73,6 +72,7 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) { } func TestCheckRepo(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, _ := repository.TestFromFixture(t, checkerTestData) chkr := checker.New(repo, false) @@ -90,6 +90,7 @@ func TestCheckRepo(t *testing.T) { } func TestMissingPack(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, be := repository.TestFromFixture(t, checkerTestData) packID := restic.TestParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6") @@ -115,6 +116,7 @@ func TestMissingPack(t *testing.T) { } func TestUnreferencedPack(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, be := repository.TestFromFixture(t, checkerTestData) // index 3f1a only references pack 60e0 @@ -142,6 +144,7 @@ func TestUnreferencedPack(t *testing.T) { } func TestUnreferencedBlobs(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, be := repository.TestFromFixture(t, checkerTestData) snapshotID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02") @@ -176,6 +179,7 @@ func TestUnreferencedBlobs(t *testing.T) { } func TestModifiedIndex(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, be := repository.TestFromFixture(t, checkerTestData) done := make(chan struct{}) @@ -215,6 +219,7 @@ func TestModifiedIndex(t *testing.T) { var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz") func TestDuplicatePacksInIndex(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("b9883c60bed42db51be171ca52f055104b6ea7cfa2bc381c05b2b1f78231280c"), `{"mac":{"k":"maQ4ILA872XnDxHVEno94A==","r":"OptMBABwkgIsMQcHME8cBw=="},"encrypt":"janrR1efN7HyQ8kOZ9zhHixooZ/e+WelH0mT4v9WskQ="}`) repo, _ := repository.TestFromFixture(t, checkerDuplicateIndexTestData) chkr := checker.New(repo, false) @@ -310,7 +315,7 @@ func (b *errorOnceBackend) Load(ctx context.Context, h backend.Handle, length in func TestCheckerModifiedData(t *testing.T) { repo, _, be := repository.TestRepositoryWithVersion(t, 0) - sn := archiver.TestSnapshot(t, repo, ".", nil) + sn := data.TestCreateSnapshot(t, repo, time.Unix(1470492820, 207401672), 2) t.Logf("archived as %v", sn.ID().Str()) errBe := &errorBackend{Backend: be} @@ -414,6 +419,7 @@ func (r *loadTreesOnceRepository) LoadBlob(ctx context.Context, bh restic.BlobHa } func TestCheckerNoDuplicateTreeDecodes(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, _ := repository.TestFromFixture(t, checkerTestData) checkRepo := &loadTreesOnceRepository{ Repository: repo, @@ -563,6 +569,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { } func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, _ := repository.TestFromFixture(t, checkerTestData) chkr := checker.New(repo, false) diff --git a/internal/data/snapshot_policy_test.go b/internal/data/snapshot_policy_test.go index 7cb943361..b74bfd667 100644 --- a/internal/data/snapshot_policy_test.go +++ b/internal/data/snapshot_policy_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "testing" "time" @@ -275,11 +276,12 @@ func TestApplyPolicy(t *testing.T) { cmpOpts := cmpopts.IgnoreUnexported(data.Snapshot{}) - if !cmp.Equal(want.Keep, keep, cmpOpts) { + // reflect.DeepEqual is faster than cmp.Equal + if !reflect.DeepEqual(want.Keep, keep) { t.Error(cmp.Diff(want.Keep, keep, cmpOpts)) } - if !cmp.Equal(want.Reasons, reasons, cmpOpts) { + if !reflect.DeepEqual(want.Reasons, reasons) { t.Error(cmp.Diff(want.Reasons, reasons, cmpOpts)) } }) diff --git a/internal/fs/fs_reader_test.go b/internal/fs/fs_reader_test.go index d56583646..a467a522f 100644 --- a/internal/fs/fs_reader_test.go +++ b/internal/fs/fs_reader_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path" + "slices" "sort" "strings" "testing" @@ -23,7 +24,8 @@ func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte test.OK(t, err) test.OK(t, f.Close()) - if !cmp.Equal(want, buf) { + // slices.Equal is much faster than cmp.Equal + if !slices.Equal(want, buf) { t.Error(cmp.Diff(want, buf)) } } diff --git a/internal/fs/node_windows_test.go b/internal/fs/node_windows_test.go index 5836dc382..174c637bd 100644 --- a/internal/fs/node_windows_test.go +++ b/internal/fs/node_windows_test.go @@ -652,6 +652,7 @@ func TestPrepareVolumeName(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + t.Parallel() isEASupported, err := checkAndStoreEASupport(tc.path) test.OK(t, err) test.Equals(t, tc.expectedEASupported, isEASupported) diff --git a/internal/repository/checker_test.go b/internal/repository/checker_test.go index 7933dca67..f344443ed 100644 --- a/internal/repository/checker_test.go +++ b/internal/repository/checker_test.go @@ -8,10 +8,11 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/klauspost/compress/zstd" - "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/data" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" @@ -33,6 +34,7 @@ func testWrapCheckPack(ctx context.Context, t *testing.T, repo *Repository, // TestGapInBlobs creates a gap in the blob list by omitting the first entry before passing it to checkPack func TestGapInBlobs(t *testing.T) { + TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, _ := TestFromFixture(t, checkerTestData) err := repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory) @@ -153,7 +155,7 @@ func setupChecker(t *testing.T, wrap func(backend.Backend) backend.Backend) *Che t.Helper() // Write a snapshot into a fresh in-memory repository. repo, be := TestRepositoryWithBackend(t, nil, 0, Options{}) - _ = archiver.TestSnapshot(t, repo, ".", nil) + data.TestCreateSnapshot(t, repo, time.Unix(1470492820, 207401672), 2) // Re-open the same backend (now containing real pack files) through // the corruption wrapper so the checker reads corrupted data. diff --git a/internal/repository/crypto/kdf_test.go b/internal/repository/crypto/kdf_test.go index 5823eb889..83d5f81d0 100644 --- a/internal/repository/crypto/kdf_test.go +++ b/internal/repository/crypto/kdf_test.go @@ -6,7 +6,7 @@ import ( ) func TestCalibrate(t *testing.T) { - params, err := Calibrate(100*time.Millisecond, 50) + params, err := Calibrate(25*time.Millisecond, 50) if err != nil { t.Fatal(err) } diff --git a/internal/repository/index/index_internal_test.go b/internal/repository/index/index_internal_test.go index 94a871ed3..c5cf2c6e6 100644 --- a/internal/repository/index/index_internal_test.go +++ b/internal/repository/index/index_internal_test.go @@ -13,11 +13,18 @@ func TestIndexOversized(t *testing.T) { // Add blobs up to indexMaxBlobs + pack.MaxHeaderEntries - 1 packID := idx.addToPacks(restic.NewRandomID()) + id := restic.NewRandomID() for i := uint(0); i < indexMaxBlobs+pack.MaxHeaderEntries-1; i++ { + // Directly modify ID to avoid benchmarking NewRandomID + id[0] = byte(i) + id[1] = byte(i >> 8) + id[2] = byte(i >> 16) + id[3] = byte(i >> 24) + idx.store(packID, pack.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, - ID: restic.NewRandomID(), + ID: id, }, Length: 100, Offset: uint(i) * 100, diff --git a/internal/repository/index/index_parallel_test.go b/internal/repository/index/index_parallel_test.go index 030fb9ea0..f69d92e84 100644 --- a/internal/repository/index/index_parallel_test.go +++ b/internal/repository/index/index_parallel_test.go @@ -2,26 +2,44 @@ package index_test import ( "context" - "path/filepath" "testing" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/repository/crypto" "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -var repoFixture = filepath.Join("..", "testdata", "test-repo.tar.gz") - func TestRepositoryForAllIndexes(t *testing.T) { - repo, _ := repository.TestFromFixture(t, repoFixture) + originalFull := index.Full + defer func() { + index.Full = originalFull + }() + index.Full = func(*index.Index) bool { return true } + + repo, unpacked, _ := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion) + + mi := index.NewMasterIndex() + for range 3 { + packID := restic.NewRandomID() + blob := pack.Blob{ + BlobHandle: restic.NewRandomBlobHandle(), + Length: uint(crypto.CiphertextLength(10)), + Offset: 0, + } + rtest.OK(t, mi.StorePack(context.TODO(), packID, pack.Blobs{blob}, unpacked)) + rtest.OK(t, mi.Flush(context.TODO(), unpacked)) + } expectedIndexIDs := restic.NewIDSet() rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { expectedIndexIDs.Insert(id) return nil })) + rtest.Assert(t, len(expectedIndexIDs) > 1, "test repo should have multiple indexes") // check that all expected indexes are loaded without errors indexIDs := restic.NewIDSet() diff --git a/internal/repository/index/master_index_test.go b/internal/repository/index/master_index_test.go index 42b24320a..97fcb0ef3 100644 --- a/internal/repository/index/master_index_test.go +++ b/internal/repository/index/master_index_test.go @@ -410,7 +410,7 @@ func BenchmarkMasterIndexGC(b *testing.B) { var ( snapshotTime = time.Unix(1470492820, 207401672) - depth = 3 + depth = 2 ) func createFilledRepo(t testing.TB, snapshots int, version uint) (*repository.Repository, restic.Unpacked[restic.FileType]) { diff --git a/internal/repository/key.go b/internal/repository/key.go index e5d1b0724..94a4184e6 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/user" + "sync" "time" "github.com/restic/restic/internal/errors" @@ -47,6 +48,9 @@ type Key struct { // calibrated on the first run of AddKey(). var params *crypto.Params +// testKeyInjection is used to speed up tests by skipping the key decryption step. +var testKeyInjection = sync.Map{} + const ( // KDFTimeout specifies the maximum runtime for the KDF. KDFTimeout = 500 * time.Millisecond @@ -63,6 +67,14 @@ func createMasterKey(ctx context.Context, s *Repository, password string) (*Key, // openKey tries do decrypt the key specified by name with the given password. func openKey(ctx context.Context, s *Repository, id restic.ID, password string) (*Key, error) { + if key, ok := testKeyInjection.Load(id); ok { + return &Key{ + master: key.(*crypto.Key), + user: key.(*crypto.Key), // not correct but good enough for testing + id: id, + }, nil + } + k, err := LoadKey(ctx, s, id) if err != nil { debug.Log("LoadKey(%v) returned error %v", id.String(), err) diff --git a/internal/repository/prune_test.go b/internal/repository/prune_test.go index 78d039b4e..cf5664104 100644 --- a/internal/repository/prune_test.go +++ b/internal/repository/prune_test.go @@ -120,12 +120,13 @@ func TestPrune(t *testing.T) { 6.) The result should be less packfiles than before */ func TestPruneSmall(t *testing.T) { + t.Parallel() seed := time.Now().UnixNano() random := rand.New(rand.NewSource(seed)) t.Logf("rand initialized with seed %d", seed) be := repository.TestBackend(t) - repo, _ := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{PackSize: repository.MinPackSize}) + repo, _ := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{PackSize: repository.MinPackSize, Compression: repository.CompressionOff}) const blobSize = 1000 * 1000 const numBlobsCreated = 55 diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 35f5c78f5..889c76049 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -919,13 +919,10 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch return err } - cfg, err := restic.CreateConfig(version) + cfg, err := restic.CreateConfig(version, chunkerPolynomial) if err != nil { return err } - if chunkerPolynomial != nil { - cfg.ChunkerPolynomial = *chunkerPolynomial - } return r.init(ctx, password, cfg) } diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index dbcecadf0..1b78f07cb 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -205,19 +205,17 @@ func testStreamPack(t *testing.T, version uint) { key := testKey(t) blobSizes := []int{ - 5522811, + 1522811, 10, 5231, 18812, 123123, - 13522811, 12301, 892242, 28616, 13351, 252287, 188883, - 3522811, 18883, } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index b830458e7..7df317752 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -28,8 +28,6 @@ import ( var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} -var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) - func TestSave(t *testing.T) { repository.TestAllVersions(t, testSavePassID) repository.TestAllVersions(t, testSaveCalculateID) @@ -45,6 +43,7 @@ func testSaveCalculateID(t *testing.T, version uint) { func testSave(t *testing.T, version uint, calculateID bool) { repo, _, _ := repository.TestRepositoryWithVersion(t, version) + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) for _, size := range testSizes { data := make([]byte, size) @@ -119,6 +118,7 @@ func testSavePackMerging(t *testing.T, targetPercentage int, expectedPacks int) // minimum pack size to speed up test PackSize: repository.MinPackSize, }) + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) var ids restic.IDs rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { @@ -162,6 +162,7 @@ func benchmarkSaveAndEncrypt(t *testing.B, version uint) { size := 4 << 20 // 4MiB data := make([]byte, size) + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) _, err := io.ReadFull(rnd, data) rtest.OK(t, err) @@ -188,6 +189,7 @@ func testLoadBlob(t *testing.T, version uint) { repo, _, _ := repository.TestRepositoryWithVersion(t, version) length := 1000000 buf := crypto.NewBlobBuffer(length) + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) _, err := io.ReadFull(rnd, buf) rtest.OK(t, err) @@ -245,6 +247,7 @@ func benchmarkLoadBlob(b *testing.B, version uint) { repo, _, _ := repository.TestRepositoryWithVersion(b, version) length := 1000000 buf := crypto.NewBlobBuffer(length) + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) @@ -286,6 +289,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) { repo, _, _ := repository.TestRepositoryWithVersion(b, version) length := 1000000 buf := crypto.NewBlobBuffer(length) + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) _, err := io.ReadFull(rnd, buf) rtest.OK(b, err) @@ -319,6 +323,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) { var repoFixture = filepath.Join("testdata", "test-repo.tar.gz") func TestRepositoryLoadIndex(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repo, _ := repository.TestFromFixture(t, repoFixture) rtest.OK(t, repo.LoadIndex(context.TODO(), restic.NoopTerminalCounterFactory)) @@ -372,6 +377,7 @@ func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length } func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) { + repository.TestInjectKey(t, restic.TestParseID("7bb3065bfb17da7430dc4dde4741d6db3dd83fdb0829500cf105755e067f879a"), `{"mac":{"k":"W1Y8bmQNJg6TAmuDt7lbpQ==","r":"r43DBmAdmwtQneoBTGAABQ=="},"encrypt":"JuZGBs6joRiLzqkyMWhmbZMLHe8+5oH6MDE5I6M8R/I="}`) repodir := rtest.Env(t, repoFixture) be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2}, t.Logf) @@ -383,9 +389,10 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) { // saveRandomDataBlobs generates random data blobs and saves them to the repository. func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) { + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) rtest.OK(t, repo.WithBlobUploader(context.TODO(), func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { for i := 0; i < num; i++ { - size := rand.Int() % sizeMax + size := rnd.Int() % sizeMax buf := make([]byte, size) _, err := io.ReadFull(rnd, buf) @@ -405,8 +412,6 @@ func TestRepositoryIncrementalIndex(t *testing.T) { func testRepositoryIncrementalIndex(t *testing.T, version uint) { repo, _, _ := repository.TestRepositoryWithVersion(t, version) - index.Full = func(*index.Index) bool { return true } - // add a few rounds of packs for j := 0; j < 5; j++ { // add some packs and write index @@ -438,7 +443,6 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { t.Errorf("pack %v listed in %d indexes\n", packID, len(ids)) } } - } func TestInvalidCompression(t *testing.T) { diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 944512a54..d44586e44 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -2,6 +2,7 @@ package repository import ( "context" + "encoding/json" "fmt" "os" "sync" @@ -54,6 +55,10 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint, o if be == nil { be = TestBackend(t) } + // Speed up tests by default + if opts.Compression == CompressionAuto { + opts.Compression = CompressionFastest + } repo, err := New(be, opts) if err != nil { @@ -126,7 +131,7 @@ func TestOpenLocal(t testing.TB, dir string) (*Repository, backend.Backend) { } func TestOpenBackend(t testing.TB, be backend.Backend) *Repository { - repo, err := New(be, Options{}) + repo, err := New(be, Options{Compression: CompressionFastest}) if err != nil { t.Fatal(err) } @@ -143,6 +148,7 @@ type VersionedTest func(t *testing.T, version uint) func TestAllVersions(t *testing.T, test VersionedTest) { for version := restic.MinRepoVersion; version <= restic.MaxRepoVersion; version++ { t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) { + t.Parallel() test(t, uint(version)) }) } @@ -189,3 +195,12 @@ func TestCheckRepo(t testing.TB, repo *Repository) { t.Error(err) } } + +func TestInjectKey(t testing.TB, keyID restic.ID, key string) { + var k crypto.Key + err := json.Unmarshal([]byte(key), &k) + if err != nil { + t.Fatal(err) + } + testKeyInjection.Store(keyID, &k) +} diff --git a/internal/restic/config.go b/internal/restic/config.go index 8af09c908..4c6c97a08 100644 --- a/internal/restic/config.go +++ b/internal/restic/config.go @@ -28,15 +28,19 @@ const StableRepoVersion = 2 // CreateConfig creates a config file with a randomly selected polynomial and // ID. -func CreateConfig(version uint) (Config, error) { +func CreateConfig(version uint, pol *chunker.Pol) (Config, error) { var ( err error cfg Config ) - cfg.ChunkerPolynomial, err = chunker.RandomPolynomial() - if err != nil { - return Config{}, errors.Wrap(err, "chunker.RandomPolynomial") + if pol == nil { + cfg.ChunkerPolynomial, err = chunker.RandomPolynomial() + if err != nil { + return Config{}, errors.Wrap(err, "chunker.RandomPolynomial") + } + } else { + cfg.ChunkerPolynomial = *pol } cfg.ID = NewRandomID().String() diff --git a/internal/restic/config_test.go b/internal/restic/config_test.go index 5a7f6b0ae..54d3db2fc 100644 --- a/internal/restic/config_test.go +++ b/internal/restic/config_test.go @@ -43,7 +43,7 @@ func TestConfig(t *testing.T) { return restic.ID{}, nil } - cfg1, err := restic.CreateConfig(restic.MaxRepoVersion) + cfg1, err := restic.CreateConfig(restic.MaxRepoVersion, nil) rtest.OK(t, err) err = restic.SaveConfig(context.TODO(), saver{save}, cfg1) diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index adb4e9cd1..5a423ea2a 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "slices" + "strings" "testing" "github.com/restic/restic/internal/errors" @@ -107,9 +108,9 @@ func newTestRepo(content []TestFile) *TestRepo { filesPathToContent := make(map[string]string) for _, file := range content { - var content string + content := strings.Builder{} for _, blob := range file.blobs { - content += blob.data + content.WriteString(blob.data) // get the pack, create as necessary var pack Pack @@ -134,7 +135,7 @@ func newTestRepo(content []TestFile) *TestRepo { packs[blob.pack] = pack } - filesPathToContent[file.name] = content + filesPathToContent[file.name] = content.String() } blobs := make(map[restic.ID][]restic.PackBlob) diff --git a/internal/test/helpers.go b/internal/test/helpers.go index 1ca5c151a..48fbf1259 100644 --- a/internal/test/helpers.go +++ b/internal/test/helpers.go @@ -195,8 +195,12 @@ func resetReadOnly(t testing.TB, dir string) { // afterwards uses os.RemoveAll() to remove the path. func RemoveAll(t testing.TB, path string) { t.Helper() - resetReadOnly(t, path) - err := os.RemoveAll(path) + var err error + err = os.RemoveAll(path) + if err != nil { + resetReadOnly(t, path) + err = os.RemoveAll(path) + } if errors.Is(err, os.ErrNotExist) { err = nil }