Add version for 0.16.4

Verifying all blobs before upload comes with a notable performance
impact. Allow users to skip it if necessary.

.github/workflows/docker.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@b4bedf8053341df3b5a9f9e0f2cf4e79e27360c6
+        uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}

.github/workflows/tests.yml

@@ -62,7 +62,7 @@ jobs:
 
     steps:
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -226,7 +226,7 @@ jobs:
 
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
@@ -244,7 +244,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
@@ -255,7 +255,7 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.52.2
+          version: v1.55.2
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get

VERSION

@@ -1 +1 @@
-0.16.2
+0.16.4

changelog/0.16.3_2024-01-14/issue-4560

@@ -0,0 +1,14 @@
+Bugfix: Improve errors for irregular files on Windows
+
+Since Go 1.21, most filesystem reparse points on Windows are considered to be
+irregular files. This caused restic to show an `error: invalid node type ""`
+error message for those files.
+
+This error message has now been improved and includes the relevant file path:
+`error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`.
+As irregular files are not required to behave like regular files, it is not
+possible to provide a generic way to back up those files.
+
+https://github.com/restic/restic/issues/4560
+https://github.com/restic/restic/pull/4620
+https://forum.restic.net/t/windows-backup-error-invalid-node-type/6875

changelog/0.16.3_2024-01-14/issue-4574

@@ -0,0 +1,11 @@
+Bugfix: Support backup of deduplicated files on Windows again
+
+With the official release builds of restic 0.16.1 and 0.16.2, it was not
+possible to back up files that were deduplicated by the corresponding
+Windows Server feature. This also applied to restic versions built using
+Go 1.21.0-1.21.4.
+
+The Go version used to build restic has now been updated to fix this.
+
+https://github.com/restic/restic/issues/4574
+https://github.com/restic/restic/pull/4621

changelog/0.16.3_2024-01-14/issue-4612

@@ -0,0 +1,11 @@
+Bugfix: Improve error handling for `rclone` backend
+
+Since restic 0.16.0, if rclone encountered an error while listing files,
+this could in rare circumstances cause restic to assume that there are no
+files. Although unlikely, this situation could result in data loss if it
+were to happen right when the `prune` command is listing existing snapshots.
+
+Error handling has now been improved to detect and work around this case.
+
+https://github.com/restic/restic/issues/4612
+https://github.com/restic/restic/pull/4618

changelog/0.16.3_2024-01-14/pull-4624

@@ -0,0 +1,11 @@
+Bugfix: Correct `restore` progress information if an error occurs
+
+If an error occurred while restoring a snapshot, this could cause the `restore`
+progress bar to show incorrect information. In addition, if a data file could
+not be loaded completely, then errors would also be reported for some already
+restored files.
+
+Error reporting of the `restore` command has now been made more accurate.
+
+https://github.com/restic/restic/pull/4624
+https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943

changelog/0.16.3_2024-01-14/pull-4626

@@ -0,0 +1,11 @@
+Bugfix: Improve reliability of restoring large files
+
+In some cases restic failed to restore large files that frequently contain the
+same file chunk. In combination with certain backends, this could result in
+network connection timeouts that caused incomplete restores.
+
+Restic now includes special handling for such file chunks to ensure reliable
+restores.
+
+https://github.com/restic/restic/pull/4626
+https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943

changelog/0.16.4_2024-02-04/issue-4529

@@ -0,0 +1,18 @@
+Enhancement: Add extra verification of data integrity before upload
+
+Hardware issues, or a bug in restic or its dependencies, could previously cause
+corruption in the files restic created and stored in the repository. Detecting
+such corruption previously required explicitly running the `check --read-data`
+or `check --read-data-subset` commands.
+
+To further ensure data integrity, even in the case of hardware issues or
+software bugs, restic now performs additional verification of the files about
+to be uploaded to the repository.
+
+These extra checks will increase CPU usage during backups. They can therefore,
+if absolutely necessary, be disabled using the `--no-extra-verify` global
+option. Please note that this should be combined with more active checking
+using the previously mentioned check commands.
+
+https://github.com/restic/restic/issues/4529
+https://github.com/restic/restic/pull/4681

changelog/0.16.4_2024-02-04/issue-4677

@@ -0,0 +1,19 @@
+Bugfix: Downgrade zstd library to fix rare data corruption at max. compression
+
+In restic 0.16.3, backups where the compression level was set to `max` (using
+`--compression max`) could in rare and very specific circumstances result in
+data corruption due to a bug in the library used for compressing data. Restic
+0.16.1 and 0.16.2 were not affected.
+
+Restic now uses the previous version of the library used to compress data, the
+same version used by restic 0.16.2. Please note that the `auto` compression
+level (which restic uses by default) was never affected, and even if you used
+`max` compression, chances of being affected by this issue are small.
+
+To check a repository for any corruption, run `restic check --read-data`. This
+will download and verify the whole repository and can be used at any time to
+completely verify the integrity of a repository. If the `check` command detects
+anomalies, follow the suggested steps.
+
+https://github.com/restic/restic/issues/4677
+https://github.com/restic/restic/pull/4679

changelog/CHANGELOG.tmpl

@@ -1,18 +1,21 @@
-{{- range $changes := . }}{{ with $changes -}}
-Changelog for restic {{ .Version }} ({{ .Date }})
-=======================================
+# Table of Contents
 
+{{ range . -}}
+  * [Changelog for {{ .Version }}](#changelog-for-restic-{{ .Version | replace "." ""}}-{{ .Date | lower -}})
+{{ end -}}
+
+{{- range $changes := . }}{{ with $changes }}
+
+# Changelog for restic {{ .Version }} ({{ .Date }})
 The following sections list the changes in restic {{ .Version }} relevant to
 restic users. The changes are ordered by importance.
 
-Summary
--------
+## Summary
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .TypeShort }} #{{ .PrimaryID }}: {{ .Title }}
 {{- end }}{{ end }}
 
-Details
--------
+## Details
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }}
 {{ range $par := .Paragraphs }}
@@ -27,6 +30,5 @@ Details
 {{ range $url := .OtherURLs }}
    {{ $url -}}
 {{ end }}
-{{ end }}{{ end }}
-
+{{ end }}{{ end -}}
 {{ end }}{{ end -}}

cmd/restic/cmd_init.go

@@ -75,7 +75,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	repo, err := ReadRepo(gopts)
+	gopts.Repo, err = ReadRepo(gopts)
 	if err != nil {
 		return err
 	}
@@ -87,7 +87,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	be, err := create(ctx, repo, gopts, gopts.extended)
+	be, err := create(ctx, gopts.Repo, gopts, gopts.extended)
 	if err != nil {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
 	}

cmd/restic/global.go

@@ -43,7 +43,7 @@ import (
 	"golang.org/x/term"
 )
 
-var version = "0.16.2"
+var version = "0.16.4"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -67,6 +67,7 @@ type GlobalOptions struct {
 	CleanupCache    bool
 	Compression     repository.CompressionMode
 	PackSize        uint
+	NoExtraVerify   bool
 
 	backend.TransportOptions
 	limiter.Limits
@@ -139,6 +140,7 @@ func init() {
 	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)")
+	f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)")
 	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
 	f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
@@ -453,8 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
 	}
 
 	s, err := repository.New(be, repository.Options{
-		Compression: opts.Compression,
-		PackSize:    opts.PackSize * 1024 * 1024,
+		Compression:   opts.Compression,
+		PackSize:      opts.PackSize * 1024 * 1024,
+		NoExtraVerify: opts.NoExtraVerify,
 	})
 	if err != nil {
 		return nil, errors.Fatal(err.Error())

cmd/restic/integration_helpers_test.go

@@ -123,9 +123,8 @@ func directoriesContentsDiff(dir1, dir2 string) string {
 				fmt.Fprintf(&out, "+%v\n", b.path)
 				b = nil
 				continue
-			} else {
-				fmt.Fprintf(&out, "%%%v\n", a.path)
 			}
+			fmt.Fprintf(&out, "%%%v\n", a.path)
 		}
 
 		a, b = nil, nil

doc/020_installation.rst

@@ -84,6 +84,12 @@ If you are using macOS, you can install restic using the
 
     $ brew install restic
 
+On Linux and macOS, you can also install it using `pkgx <https://pkgx.sh/>`__:
+
+.. code-block:: console
+
+    $ pkgx install restic
+
 You may also install it using `MacPorts <https://www.macports.org/>`__:
 
 .. code-block:: console

doc/045_working_with_repos.rst

@@ -120,7 +120,7 @@ be skipped by later copy runs.
 The source repository is specified with ``--from-repo`` or can be read
 from a file specified via ``--from-repository-file``. Both of these options
 can also be set as environment variables ``$RESTIC_FROM_REPOSITORY`` or
-``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the destination repository
+``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the source repository
 the password can be read from a file ``--from-password-file`` or from a command
 ``--from-password-command``.
 Alternatively the environment variables ``$RESTIC_FROM_PASSWORD_COMMAND`` and

doc/047_tuning_backup_parameters.rst

@@ -60,6 +60,20 @@ only applied for the single run of restic. The option can also be set via the en
 variable ``RESTIC_COMPRESSION``.
 
 
+Data Verification
+=================
+
+To prevent the upload of corrupted data to the repository, which can happen due
+to hardware issues or software bugs, restic verifies that generated files can
+be decoded and contain the correct data beforehand. This increases the CPU usage
+during backups. If necessary, you can disable this verification using the
+``--no-extra-verify`` option of the ``backup`` command. However, in this case
+you should verify the repository integrity more actively using
+``restic check --read-data`` (or the similar ``--read-data-subset`` option).
+Otherwise, data corruption due to hardware issues or software bugs might go
+unnoticed.
+
+
 File Read Concurrency
 =====================
 

doc/060_forget.rst

@@ -207,10 +207,13 @@ The ``forget`` command accepts the following policy options:
     They also only count hours/days/weeks/etc which have one or more snapshots.
     A value of ``-1`` will be interpreted as "forever", i.e. "keep all".
 
-.. note:: All duration related options (``--keep-{within,-*}``) ignore snapshots
+.. note:: All duration related options (``--keep-{within-,}*``) ignore snapshots
     with a timestamp in the future (relative to when the ``forget`` command is
     run) and these snapshots will hence not be removed.
 
+.. note:: If there are not enough snapshots to keep one for each duration related
+    ``--keep-{within-,}*`` option, the oldest snapshot is kept additionally.
+
 .. note:: Specifying ``--keep-tag ''`` will match untagged snapshots only.
 
 When ``forget`` is run with a policy, restic first loads the list of all snapshots

doc/075_scripting.rst

@@ -556,7 +556,7 @@ The snapshots command returns a single JSON object, an array with objects of the
 stats
 -----
 
-The snapshots command returns a single JSON object.
+The stats command returns a single JSON object.
 
 +------------------------------+-----------------------------------------------------+
 | ``total_size``               | Repository size in bytes                            |

doc/bash-completion.sh

@@ -488,6 +488,7 @@ _restic_backup()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -560,6 +561,7 @@ _restic_cache()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -624,6 +626,7 @@ _restic_cat()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -696,6 +699,7 @@ _restic_check()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -794,6 +798,7 @@ _restic_copy()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -860,6 +865,7 @@ _restic_diff()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -944,6 +950,7 @@ _restic_dump()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1058,6 +1065,7 @@ _restic_find()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1228,6 +1236,7 @@ _restic_forget()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1312,6 +1321,7 @@ _restic_generate()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1372,6 +1382,7 @@ _restic_help()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1463,6 +1474,7 @@ _restic_init()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1539,6 +1551,7 @@ _restic_key()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1603,6 +1616,7 @@ _restic_list()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1689,6 +1703,7 @@ _restic_ls()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1757,6 +1772,7 @@ _restic_migrate()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1849,6 +1865,7 @@ _restic_mount()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1935,6 +1952,7 @@ _restic_prune()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -1999,6 +2017,7 @@ _restic_recover()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2059,6 +2078,7 @@ _restic_repair_help()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2126,6 +2146,7 @@ _restic_repair_index()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2190,6 +2211,7 @@ _restic_repair_packs()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2274,6 +2296,7 @@ _restic_repair_snapshots()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2342,6 +2365,7 @@ _restic_repair()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2450,6 +2474,7 @@ _restic_restore()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2552,6 +2577,7 @@ _restic_rewrite()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2620,6 +2646,7 @@ _restic_self-update()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2712,6 +2739,7 @@ _restic_snapshots()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2794,6 +2822,7 @@ _restic_stats()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2884,6 +2913,7 @@ _restic_tag()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -2950,6 +2980,7 @@ _restic_unlock()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -3014,6 +3045,7 @@ _restic_version()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")
@@ -3106,6 +3138,7 @@ _restic_root_command()
     flags+=("--limit-upload=")
     two_word_flags+=("--limit-upload")
     flags+=("--no-cache")
+    flags+=("--no-extra-verify")
     flags+=("--no-lock")
     flags+=("--option=")
     two_word_flags+=("--option")

doc/man/restic-backup.1

@@ -171,6 +171,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-cache.1

@@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-cat.1

@@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-check.1

@@ -85,6 +85,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-copy.1

@@ -109,6 +109,10 @@ new destination repository using the "init" command.
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-diff.1

@@ -93,6 +93,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-dump.1

@@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-find.1

@@ -117,6 +117,10 @@ It can also be used to search for restic blobs or trees for troubleshooting.
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-forget.1

@@ -179,6 +179,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-generate.1

@@ -89,6 +89,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-init.1

@@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-key.1

@@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-list.1

@@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-ls.1

@@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-migrate.1

@@ -74,6 +74,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-mount.1

@@ -144,6 +144,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-prune.1

@@ -97,6 +97,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-recover.1

@@ -70,6 +70,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-repair-index.1

@@ -73,6 +73,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-repair-packs.1

@@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-repair-snapshots.1

@@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-repair.1

@@ -63,6 +63,10 @@ Repair the repository
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-restore.1

@@ -117,6 +117,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-rewrite.1

@@ -121,6 +121,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-self-update.1

@@ -75,6 +75,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-snapshots.1

@@ -92,6 +92,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-stats.1

@@ -114,6 +114,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-tag.1

@@ -99,6 +99,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-unlock.1

@@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic-version.1

@@ -69,6 +69,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/man/restic.1

@@ -65,6 +65,10 @@ The full documentation can be found at https://restic.readthedocs.io/ .
 \fB--no-cache\fP[=false]
 	do not use a local cache
 
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
 .PP
 \fB--no-lock\fP[=false]
 	do not lock the repository, this allows some operations on read-only repositories

doc/manual_rest.rst

@@ -148,11 +148,11 @@ command:
       -v, --verbose                    be verbose (specify multiple times or a level using --verbose=n, max level/times is 2)
 
 Subcommands that support showing progress information such as ``backup``,
-``check`` and ``prune`` will do so unless the quiet flag ``-q`` or
-``--quiet`` is set. When running from a non-interactive console progress
-reporting is disabled by default to not fill your logs. For interactive
-and non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS``
-can be used to control the frequency of progress reporting. Use for example
+``restore``, ``check`` and ``prune`` will do so unless the quiet flag ``-q``
+or ``--quiet`` is set. When running from a non-interactive console progress
+reporting is disabled by default to not fill your logs. For interactive and
+non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS`` can
+be used to control the frequency of progress reporting. Use for example
 ``0.016666`` to only update the progress once per minute.
 
 Additionally, on Unix systems if ``restic`` receives a SIGUSR1 signal the

go.mod

@@ -1,8 +1,8 @@
 module github.com/restic/restic
 
 require (
-	cloud.google.com/go/storage v1.33.0
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0
+	cloud.google.com/go/storage v1.34.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
 	github.com/Backblaze/blazer v0.6.1
@@ -13,8 +13,8 @@ require (
 	github.com/go-ole/go-ole v1.3.0
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/klauspost/compress v1.17.2
-	github.com/minio/minio-go/v7 v7.0.63
+	github.com/klauspost/compress v1.17.4
+	github.com/minio/minio-go/v7 v7.0.66
 	github.com/minio/sha256-simd v1.0.1
 	github.com/ncw/swift/v2 v2.0.2
 	github.com/pkg/errors v0.9.1
@@ -25,23 +25,25 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	go.uber.org/automaxprocs v1.5.3
-	golang.org/x/crypto v0.14.0
-	golang.org/x/net v0.17.0
-	golang.org/x/oauth2 v0.13.0
-	golang.org/x/sync v0.4.0
-	golang.org/x/sys v0.13.0
-	golang.org/x/term v0.13.0
-	golang.org/x/text v0.13.0
-	golang.org/x/time v0.3.0
-	google.golang.org/api v0.148.0
+	golang.org/x/crypto v0.17.0
+	golang.org/x/net v0.19.0
+	golang.org/x/oauth2 v0.15.0
+	golang.org/x/sync v0.5.0
+	golang.org/x/sys v0.15.0
+	golang.org/x/term v0.15.0
+	golang.org/x/text v0.14.0
+	golang.org/x/time v0.5.0
+	google.golang.org/api v0.149.0
 )
 
+replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2
+
 require (
 	cloud.google.com/go v0.110.9 // indirect
 	cloud.google.com/go/compute v1.23.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -51,12 +53,12 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/google/uuid v1.3.1 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect

go.sum

@@ -7,14 +7,14 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc=
 cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE=
-cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
-cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw=
+cloud.google.com/go/storage v1.34.0 h1:9KHBBTbaHPsNxO043SFmH3pMojjZiW+BFl9H41L7xjk=
+cloud.google.com/go/storage v1.34.0/go.mod h1:Eji+S0CCQebjsiXxyIvPItC3BN3zWsdJjWfHfoLblgY=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 h1:TuEMD+E+1aTjjLICGQOW6vLe8UWES7kopac9mUXL56Y=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
@@ -95,10 +95,10 @@ github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0Z
 github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
 github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -111,8 +111,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
 github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
-github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -122,8 +122,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
-github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
+github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
+github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -183,8 +183,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -202,18 +202,18 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
+golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -229,22 +229,22 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -258,8 +258,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs=
-google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU=
+google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY=
+google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=

internal/archiver/archiver.go

@@ -2,10 +2,12 @@ package archiver
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path"
 	"runtime"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/debug"
@@ -168,6 +170,11 @@ func (arch *Archiver) error(item string, err error) error {
 		return err
 	}
 
+	// not all errors include the filepath, thus add it if it is missing
+	if !strings.Contains(err.Error(), item) {
+		err = fmt.Errorf("%v: %w", item, err)
+	}
+
 	errf := arch.Error(item, err)
 	if err != errf {
 		debug.Log("item %v: error was filtered by handler, before: %q, after: %v", item, err, errf)
@@ -183,7 +190,10 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo)
 	}
 	// overwrite name to match that within the snapshot
 	node.Name = path.Base(snPath)
-	return node, errors.WithStack(err)
+	if err != nil {
+		return node, fmt.Errorf("nodeFromFileInfo %v: %w", filename, err)
+	}
+	return node, err
 }
 
 // loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned.

internal/archiver/archiver_test.go

@@ -1879,7 +1879,7 @@ func TestArchiverContextCanceled(t *testing.T) {
 	})
 
 	// Ensure that the archiver itself reports the canceled context and not just the backend
-	repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0)
+	repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{})
 
 	back := restictest.Chdir(t, tempdir)
 	defer back()

internal/backend/rest/rest.go

@@ -328,8 +328,13 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
 	}
 
 	if resp.StatusCode == http.StatusNotFound {
-		// ignore missing directories
-		return nil
+		if !strings.HasPrefix(resp.Header.Get("Server"), "rclone/") {
+			// ignore missing directories, unless the server is rclone. rclone
+			// already ignores missing directories, but misuses "not found" to
+			// report certain internal errors, see
+			// https://github.com/rclone/rclone/pull/7550 for details.
+			return nil
+		}
 	}
 
 	if resp.StatusCode != http.StatusOK {

internal/fuse/snapshots_dir.go

@@ -110,9 +110,8 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
 			return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot)
 		} else if entry.snapshot != nil {
 			return newDirFromSnapshot(d.root, inode, entry.snapshot)
-		} else {
-			return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
 		}
+		return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
 	}
 
 	return nil, syscall.ENOENT

internal/migrations/upgrade_repo_v2_test.go

@@ -69,7 +69,7 @@ func TestUpgradeRepoV2Failure(t *testing.T) {
 		Backend:                   be,
 	}
 
-	repo := repository.TestRepositoryWithBackend(t, be, 1)
+	repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{})
 	if repo.Config().Version != 1 {
 		t.Fatal("test repo has wrong version")
 	}

internal/pack/pack.go

@@ -1,6 +1,7 @@
 package pack
 
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
 	"fmt"
@@ -74,7 +75,7 @@ func (p *Packer) Finalize() error {
 	p.m.Lock()
 	defer p.m.Unlock()
 
-	header, err := p.makeHeader()
+	header, err := makeHeader(p.blobs)
 	if err != nil {
 		return err
 	}
@@ -83,6 +84,12 @@ func (p *Packer) Finalize() error {
 	nonce := crypto.NewRandomNonce()
 	encryptedHeader = append(encryptedHeader, nonce...)
 	encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil)
+	encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader)))
+
+	if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil {
+		//nolint:revive // ignore linter warnings about error message spelling
+		return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err)
+	}
 
 	// append the header
 	n, err := p.wr.Write(encryptedHeader)
@@ -90,18 +97,33 @@ func (p *Packer) Finalize() error {
 		return errors.Wrap(err, "Write")
 	}
 
-	hdrBytes := len(encryptedHeader)
-	if n != hdrBytes {
+	if n != len(encryptedHeader) {
 		return errors.New("wrong number of bytes written")
 	}
+	p.bytes += uint(len(encryptedHeader))
 
-	// write length
-	err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes))
+	return nil
+}
+
+func verifyHeader(k *crypto.Key, header []byte, expected []restic.Blob) error {
+	// do not offer a way to skip the pack header verification, as pack headers are usually small enough
+	// to not result in a significant performance impact
+
+	decoded, hdrSize, err := List(k, bytes.NewReader(header), int64(len(header)))
 	if err != nil {
-		return errors.Wrap(err, "binary.Write")
+		return fmt.Errorf("header decoding failed: %w", err)
+	}
+	if hdrSize != uint32(len(header)) {
+		return fmt.Errorf("unexpected header size %v instead of %v", hdrSize, len(header))
+	}
+	if len(decoded) != len(expected) {
+		return fmt.Errorf("pack header size mismatch")
+	}
+	for i := 0; i < len(decoded); i++ {
+		if decoded[i] != expected[i] {
+			return fmt.Errorf("pack header entry mismatch got %v instead of %v", decoded[i], expected[i])
+		}
 	}
-	p.bytes += uint(hdrBytes + binary.Size(uint32(0)))
-
 	return nil
 }
 
@@ -111,10 +133,10 @@ func (p *Packer) HeaderOverhead() int {
 }
 
 // makeHeader constructs the header for p.
-func (p *Packer) makeHeader() ([]byte, error) {
-	buf := make([]byte, 0, len(p.blobs)*int(entrySize))
+func makeHeader(blobs []restic.Blob) ([]byte, error) {
+	buf := make([]byte, 0, len(blobs)*int(entrySize))
 
-	for _, b := range p.blobs {
+	for _, b := range blobs {
 		switch {
 		case b.Type == restic.DataBlob && b.UncompressedLength == 0:
 			buf = append(buf, 0)

internal/pack/pack_internal_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"io"
+	"strings"
 	"testing"
 
 	"github.com/restic/restic/internal/crypto"
@@ -177,3 +178,60 @@ func TestReadRecords(t *testing.T) {
 		}
 	}
 }
+
+func TestUnpackedVerification(t *testing.T) {
+	// create random keys
+	k := crypto.NewRandomKey()
+	blobs := []restic.Blob{
+		{
+			BlobHandle:         restic.NewRandomBlobHandle(),
+			Length:             42,
+			Offset:             0,
+			UncompressedLength: 2 * 42,
+		},
+	}
+
+	type DamageType string
+	const (
+		damageData       DamageType = "data"
+		damageCiphertext DamageType = "ciphertext"
+		damageLength     DamageType = "length"
+	)
+
+	for _, test := range []struct {
+		damage DamageType
+		msg    string
+	}{
+		{"", ""},
+		{damageData, "pack header entry mismatch"},
+		{damageCiphertext, "ciphertext verification failed"},
+		{damageLength, "header decoding failed"},
+	} {
+		header, err := makeHeader(blobs)
+		rtest.OK(t, err)
+
+		if test.damage == damageData {
+			header[8] ^= 0x42
+		}
+
+		encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header)))
+		nonce := crypto.NewRandomNonce()
+		encryptedHeader = append(encryptedHeader, nonce...)
+		encryptedHeader = k.Seal(encryptedHeader, nonce, header, nil)
+		encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader)))
+
+		if test.damage == damageCiphertext {
+			encryptedHeader[8] ^= 0x42
+		}
+		if test.damage == damageLength {
+			encryptedHeader[len(encryptedHeader)-1] ^= 0x42
+		}
+
+		err = verifyHeader(k, encryptedHeader, blobs)
+		if test.msg == "" {
+			rtest.Assert(t, err == nil, "expected no error, got %v", err)
+		} else {
+			rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err)
+		}
+	}
+}

internal/repository/fuzz_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/restic/restic/internal/backend/mem"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
 )
@@ -19,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) {
 		}
 
 		id := restic.Hash(blob)
-		repo := TestRepositoryWithBackend(t, mem.New(), 2)
+		repo := TestRepositoryWithVersion(t, 2)
 
 		var wg errgroup.Group
 		repo.StartPackUploader(context.TODO(), &wg)

internal/repository/repack_test.go

@@ -346,7 +346,8 @@ func TestRepackWrongBlob(t *testing.T) {
 }
 
 func testRepackWrongBlob(t *testing.T, version uint) {
-	repo := repository.TestRepositoryWithVersion(t, version)
+	// disable verification to allow adding corrupted blobs to the repository
+	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true})
 
 	seed := time.Now().UnixNano()
 	rand.Seed(seed)
@@ -371,7 +372,8 @@ func TestRepackBlobFallback(t *testing.T) {
 }
 
 func testRepackBlobFallback(t *testing.T, version uint) {
-	repo := repository.TestRepositoryWithVersion(t, version)
+	// disable verification to allow adding corrupted blobs to the repository
+	repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true})
 
 	seed := time.Now().UnixNano()
 	rand.Seed(seed)

internal/repository/repository.go

@@ -59,8 +59,9 @@ type Repository struct {
 }
 
 type Options struct {
-	Compression CompressionMode
-	PackSize    uint
+	Compression   CompressionMode
+	PackSize      uint
+	NoExtraVerify bool
 }
 
 // CompressionMode configures if data should be compressed.
@@ -423,6 +424,11 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	// encrypt blob
 	ciphertext = r.key.Seal(ciphertext, nonce, data, nil)
 
+	if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil {
+		//nolint:revive // ignore linter warnings about error message spelling
+		return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err)
+	}
+
 	// find suitable packer and add blob
 	var pm *packerManager
 
@@ -438,6 +444,31 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data
 	return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength)
 }
 
+func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error {
+	if r.opts.NoExtraVerify {
+		return nil
+	}
+
+	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
+	plaintext, err := r.key.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return fmt.Errorf("decryption failed: %w", err)
+	}
+	if uncompressedLength != 0 {
+		// DecodeAll will allocate a slice if it is not large enough since it
+		// knows the decompressed size (because we're using EncodeAll)
+		plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, nil)
+		if err != nil {
+			return fmt.Errorf("decompression failed: %w", err)
+		}
+	}
+	if !restic.Hash(plaintext).Equal(id) {
+		return errors.New("hash mismatch")
+	}
+
+	return nil
+}
+
 func (r *Repository) compressUnpacked(p []byte) ([]byte, error) {
 	// compression is only available starting from version 2
 	if r.cfg.Version < 2 {
@@ -474,7 +505,8 @@ func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) {
 
 // SaveUnpacked encrypts data and stores it in the backend. Returned is the
 // storage hash.
-func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) {
+func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (id restic.ID, err error) {
+	p := buf
 	if t != restic.ConfigFile {
 		p, err = r.compressUnpacked(p)
 		if err != nil {
@@ -489,6 +521,11 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 
 	ciphertext = r.key.Seal(ciphertext, nonce, p, nil)
 
+	if err := r.verifyUnpacked(ciphertext, t, buf); err != nil {
+		//nolint:revive // ignore linter warnings about error message spelling
+		return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err)
+	}
+
 	if t == restic.ConfigFile {
 		id = restic.ID{}
 	} else {
@@ -506,6 +543,29 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by
 	return id, nil
 }
 
+func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error {
+	if r.opts.NoExtraVerify {
+		return nil
+	}
+
+	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
+	plaintext, err := r.key.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return fmt.Errorf("decryption failed: %w", err)
+	}
+	if t != restic.ConfigFile {
+		plaintext, err = r.decompressUnpacked(plaintext)
+		if err != nil {
+			return fmt.Errorf("decompression failed: %w", err)
+		}
+	}
+
+	if !bytes.Equal(plaintext, expected) {
+		return errors.New("data mismatch")
+	}
+	return nil
+}
+
 // Flush saves all remaining packs and the index
 func (r *Repository) Flush(ctx context.Context) error {
 	if err := r.flushPacks(ctx); err != nil {
@@ -887,9 +947,9 @@ type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset
 const maxUnusedRange = 4 * 1024 * 1024
 
 // StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to
-// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. In
-// case of download errors handleBlobFn might be called multiple times for the same blob. If the
-// callback returns an error, then StreamPack will abort and not retry it.
+// the handleBlobFn callback or an error if decryption failed or the blob hash does not match.
+// handleBlobFn is never called multiple times for the same blob. If the callback returns an error,
+// then StreamPack will abort and not retry it.
 func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
 	if len(blobs) == 0 {
 		// nothing to do
@@ -951,7 +1011,9 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key,
 		currentBlobEnd := dataStart
 		var buf []byte
 		var decode []byte
-		for _, entry := range blobs {
+		for len(blobs) > 0 {
+			entry := blobs[0]
+
 			skipBytes := int(entry.Offset - currentBlobEnd)
 			if skipBytes < 0 {
 				return errors.Errorf("overlapping blobs in pack %v", packID)
@@ -1014,6 +1076,8 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key,
 				cancel()
 				return backoff.Permanent(err)
 			}
+			// ensure that each blob is only passed once to handleBlobFn
+			blobs = blobs[1:]
 		}
 		return nil
 	})

internal/repository/repository_internal_test.go

@@ -3,8 +3,10 @@ package repository
 import (
 	"math/rand"
 	"sort"
+	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -72,3 +74,101 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) {
 		sortCachedPacksFirst(cache, cpy[:])
 	}
 }
+
+func TestBlobVerification(t *testing.T) {
+	repo := TestRepository(t).(*Repository)
+
+	type DamageType string
+	const (
+		damageData       DamageType = "data"
+		damageCompressed DamageType = "compressed"
+		damageCiphertext DamageType = "ciphertext"
+	)
+
+	for _, test := range []struct {
+		damage DamageType
+		msg    string
+	}{
+		{"", ""},
+		{damageData, "hash mismatch"},
+		{damageCompressed, "decompression failed"},
+		{damageCiphertext, "ciphertext verification failed"},
+	} {
+		plaintext := rtest.Random(800, 1234)
+		id := restic.Hash(plaintext)
+		if test.damage == damageData {
+			plaintext[42] ^= 0x42
+		}
+
+		uncompressedLength := uint(len(plaintext))
+		plaintext = repo.getZstdEncoder().EncodeAll(plaintext, nil)
+
+		if test.damage == damageCompressed {
+			plaintext = plaintext[:len(plaintext)-8]
+		}
+
+		nonce := crypto.NewRandomNonce()
+		ciphertext := append([]byte{}, nonce...)
+		ciphertext = repo.Key().Seal(ciphertext, nonce, plaintext, nil)
+
+		if test.damage == damageCiphertext {
+			ciphertext[42] ^= 0x42
+		}
+
+		err := repo.verifyCiphertext(ciphertext, int(uncompressedLength), id)
+		if test.msg == "" {
+			rtest.Assert(t, err == nil, "expected no error, got %v", err)
+		} else {
+			rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err)
+		}
+	}
+}
+
+func TestUnpackedVerification(t *testing.T) {
+	repo := TestRepository(t).(*Repository)
+
+	type DamageType string
+	const (
+		damageData       DamageType = "data"
+		damageCompressed DamageType = "compressed"
+		damageCiphertext DamageType = "ciphertext"
+	)
+
+	for _, test := range []struct {
+		damage DamageType
+		msg    string
+	}{
+		{"", ""},
+		{damageData, "data mismatch"},
+		{damageCompressed, "decompression failed"},
+		{damageCiphertext, "ciphertext verification failed"},
+	} {
+		plaintext := rtest.Random(800, 1234)
+		orig := append([]byte{}, plaintext...)
+		if test.damage == damageData {
+			plaintext[42] ^= 0x42
+		}
+
+		compressed := []byte{2}
+		compressed = repo.getZstdEncoder().EncodeAll(plaintext, compressed)
+
+		if test.damage == damageCompressed {
+			compressed = compressed[:len(compressed)-8]
+		}
+
+		nonce := crypto.NewRandomNonce()
+		ciphertext := append([]byte{}, nonce...)
+		ciphertext = repo.Key().Seal(ciphertext, nonce, compressed, nil)
+
+		if test.damage == damageCiphertext {
+			ciphertext[42] ^= 0x42
+		}
+
+		err := repo.verifyUnpacked(ciphertext, restic.IndexFile, orig)
+		if test.msg == "" {
+			rtest.Assert(t, err == nil, "expected no error, got %v", err)
+		} else {
+			rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err)
+		}
+	}
+}

internal/repository/repository_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -14,6 +15,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/google/go-cmp/cmp"
 	"github.com/klauspost/compress/zstd"
 	"github.com/restic/restic/internal/backend/local"
@@ -528,7 +530,9 @@ func testStreamPack(t *testing.T, version uint) {
 	packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress)
 
 	loadCalls := 0
-	load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+	shortFirstLoad := false
+
+	loadBytes := func(length int, offset int64) []byte {
 		data := packfile
 
 		if offset > int64(len(data)) {
@@ -540,32 +544,56 @@ func testStreamPack(t *testing.T, version uint) {
 		if length > len(data) {
 			length = len(data)
 		}
+		if shortFirstLoad {
+			length /= 2
+			shortFirstLoad = false
+		}
+
+		return data[:length]
+	}
+
+	load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+		data := loadBytes(length, offset)
+		if shortFirstLoad {
+			data = data[:len(data)/2]
+			shortFirstLoad = false
+		}
 
-		data = data[:length]
 		loadCalls++
 
-		return fn(bytes.NewReader(data))
+		err := fn(bytes.NewReader(data))
+		if err == nil {
+			return nil
+		}
+		var permanent *backoff.PermanentError
+		if errors.As(err, &permanent) {
+			return err
+		}
 
+		// retry loading once
+		return fn(bytes.NewReader(loadBytes(length, offset)))
 	}
 
 	// first, test regular usage
 	t.Run("regular", func(t *testing.T) {
 		tests := []struct {
-			blobs []restic.Blob
-			calls int
+			blobs          []restic.Blob
+			calls          int
+			shortFirstLoad bool
 		}{
-			{packfileBlobs[1:2], 1},
-			{packfileBlobs[2:5], 1},
-			{packfileBlobs[2:8], 1},
+			{packfileBlobs[1:2], 1, false},
+			{packfileBlobs[2:5], 1, false},
+			{packfileBlobs[2:8], 1, false},
 			{[]restic.Blob{
 				packfileBlobs[0],
 				packfileBlobs[4],
 				packfileBlobs[2],
-			}, 1},
+			}, 1, false},
 			{[]restic.Blob{
 				packfileBlobs[0],
 				packfileBlobs[len(packfileBlobs)-1],
-			}, 2},
+			}, 2, false},
+			{packfileBlobs[:], 1, true},
 		}
 
 		for _, test := range tests {
@@ -592,6 +620,7 @@ func testStreamPack(t *testing.T, version uint) {
 				}
 
 				loadCalls = 0
+				shortFirstLoad = test.shortFirstLoad
 				err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob)
 				if err != nil {
 					t.Fatal(err)
@@ -604,6 +633,7 @@ func testStreamPack(t *testing.T, version uint) {
 			})
 		}
 	})
+	shortFirstLoad = false
 
 	// next, test invalid uses, which should return an error
 	t.Run("invalid", func(t *testing.T) {

internal/repository/testing.go

@@ -43,7 +43,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
 // TestRepositoryWithBackend returns a repository initialized with a test
 // password. If be is nil, an in-memory backend is used. A constant polynomial
 // is used for the chunker and low-security test parameters.
-func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository {
+func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint, opts Options) restic.Repository {
 	t.Helper()
 	TestUseLowSecurityKDFParameters(t)
 	restic.TestDisableCheckPolynomial(t)
@@ -52,7 +52,7 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) re
 		be = TestBackend(t)
 	}
 
-	repo, err := New(be, Options{})
+	repo, err := New(be, opts)
 	if err != nil {
 		t.Fatalf("TestRepository(): new repo failed: %v", err)
 	}
@@ -78,6 +78,7 @@ func TestRepository(t testing.TB) restic.Repository {
 func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
 	t.Helper()
 	dir := os.Getenv("RESTIC_TEST_REPO")
+	opts := Options{}
 	if dir != "" {
 		_, err := os.Stat(dir)
 		if err != nil {
@@ -85,7 +86,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
 			if err != nil {
 				t.Fatalf("error creating local backend at %v: %v", dir, err)
 			}
-			return TestRepositoryWithBackend(t, be, version)
+			return TestRepositoryWithBackend(t, be, version, opts)
 		}
 
 		if err == nil {
@@ -93,7 +94,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
 		}
 	}
 
-	return TestRepositoryWithBackend(t, nil, version)
+	return TestRepositoryWithBackend(t, nil, version, opts)
 }
 
 // TestOpenLocal opens a local repository.

internal/restic/lock_test.go

@@ -65,7 +65,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, len
 
 func TestMultipleLockFailure(t *testing.T) {
 	be := &failLockLoadingBackend{Backend: mem.New()}
-	repo := repository.TestRepositoryWithBackend(t, be, 0)
+	repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{})
 	restic.TestSetLockTimeout(t, 5*time.Millisecond)
 
 	lock1, err := restic.NewLock(context.TODO(), repo)

internal/restic/node.go

@@ -109,7 +109,7 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
 }
 
 func nodeTypeFromFileInfo(fi os.FileInfo) string {
-	switch fi.Mode() & (os.ModeType | os.ModeCharDevice) {
+	switch fi.Mode() & os.ModeType {
 	case 0:
 		return "file"
 	case os.ModeDir:
@@ -124,6 +124,8 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
 		return "fifo"
 	case os.ModeSocket:
 		return "socket"
+	case os.ModeIrregular:
+		return "irregular"
 	}
 
 	return ""
@@ -622,7 +624,7 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
 	case "fifo":
 	case "socket":
 	default:
-		return errors.Errorf("invalid node type %q", node.Type)
+		return errors.Errorf("unsupported file type %q", node.Type)
 	}
 
 	return node.fillExtendedAttributes(path)

internal/restorer/filerestorer.go

@@ -197,19 +197,20 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 	return wg.Wait()
 }
 
-func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
+type blobToFileOffsetsMapping map[restic.ID]struct {
+	files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file
+	blob  restic.Blob
+}
 
+func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
 	// calculate blob->[]files->[]offsets mappings
-	blobs := make(map[restic.ID]struct {
-		files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file
-	})
-	var blobList []restic.Blob
+	blobs := make(blobToFileOffsetsMapping)
 	for file := range pack.files {
 		addBlob := func(blob restic.Blob, fileOffset int64) {
 			blobInfo, ok := blobs[blob.ID]
 			if !ok {
 				blobInfo.files = make(map[*fileInfo][]int64)
-				blobList = append(blobList, blob)
+				blobInfo.blob = blob
 				blobs[blob.ID] = blobInfo
 			}
 			blobInfo.files[file] = append(blobInfo.files[file], fileOffset)
@@ -239,65 +240,120 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
 		}
 	}
 
-	sanitizeError := func(file *fileInfo, err error) error {
-		if err != nil {
-			err = r.Error(file.location, err)
+	// track already processed blobs for precise error reporting
+	processedBlobs := restic.NewBlobSet()
+	for _, entry := range blobs {
+		occurrences := 0
+		for _, offsets := range entry.files {
+			occurrences += len(offsets)
+		}
+		// With a maximum blob size of 8MB, the normal blob streaming has to write
+		// at most 800MB for a single blob. This should be short enough to avoid
+		// network connection timeouts. Based on a quick test, a limit of 100 only
+		// selects a very small number of blobs (the number of references per blob
+		// - aka. `count` - seem to follow a expontential distribution)
+		if occurrences > 100 {
+			// process frequently referenced blobs first as these can take a long time to write
+			// which can cause backend connections to time out
+			delete(blobs, entry.blob.ID)
+			partialBlobs := blobToFileOffsetsMapping{entry.blob.ID: entry}
+			err := r.downloadBlobs(ctx, pack.id, partialBlobs, processedBlobs)
+			if err := r.reportError(blobs, processedBlobs, err); err != nil {
+				return err
+			}
 		}
-		return err
 	}
 
-	err := repository.StreamPack(ctx, r.packLoader, r.key, pack.id, blobList, func(h restic.BlobHandle, blobData []byte, err error) error {
-		blob := blobs[h.ID]
-		if err != nil {
-			for file := range blob.files {
-				if errFile := sanitizeError(file, err); errFile != nil {
-					return errFile
+	if len(blobs) == 0 {
+		return nil
+	}
+
+	err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs)
+	return r.reportError(blobs, processedBlobs, err)
+}
+
+func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error {
+	if err != nil {
+		err = r.Error(file.location, err)
+	}
+	return err
+}
+
+func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet, err error) error {
+	if err == nil {
+		return nil
+	}
+
+	// only report error for not yet processed blobs
+	affectedFiles := make(map[*fileInfo]struct{})
+	for _, entry := range blobs {
+		if processedBlobs.Has(entry.blob.BlobHandle) {
+			continue
+		}
+		for file := range entry.files {
+			affectedFiles[file] = struct{}{}
+		}
+	}
+
+	for file := range affectedFiles {
+		if errFile := r.sanitizeError(file, err); errFile != nil {
+			return errFile
+		}
+	}
+	return nil
+}
+
+func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID,
+	blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error {
+
+	blobList := make([]restic.Blob, 0, len(blobs))
+	for _, entry := range blobs {
+		blobList = append(blobList, entry.blob)
+	}
+	return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList,
+		func(h restic.BlobHandle, blobData []byte, err error) error {
+			processedBlobs.Insert(h)
+			blob := blobs[h.ID]
+			if err != nil {
+				for file := range blob.files {
+					if errFile := r.sanitizeError(file, err); errFile != nil {
+						return errFile
+					}
+				}
+				return nil
+			}
+			for file, offsets := range blob.files {
+				for _, offset := range offsets {
+					writeToFile := func() error {
+						// this looks overly complicated and needs explanation
+						// two competing requirements:
+						// - must create the file once and only once
+						// - should allow concurrent writes to the file
+						// so write the first blob while holding file lock
+						// write other blobs after releasing the lock
+						createSize := int64(-1)
+						file.lock.Lock()
+						if file.inProgress {
+							file.lock.Unlock()
+						} else {
+							defer file.lock.Unlock()
+							file.inProgress = true
+							createSize = file.size
+						}
+						writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse)
+
+						if r.progress != nil {
+							r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
+						}
+
+						return writeErr
+					}
+					err := r.sanitizeError(file, writeToFile())
+					if err != nil {
+						return err
+					}
 				}
 			}
 			return nil
-		}
-		for file, offsets := range blob.files {
-			for _, offset := range offsets {
-				writeToFile := func() error {
-					// this looks overly complicated and needs explanation
-					// two competing requirements:
-					// - must create the file once and only once
-					// - should allow concurrent writes to the file
-					// so write the first blob while holding file lock
-					// write other blobs after releasing the lock
-					createSize := int64(-1)
-					file.lock.Lock()
-					if file.inProgress {
-						file.lock.Unlock()
-					} else {
-						defer file.lock.Unlock()
-						file.inProgress = true
-						createSize = file.size
-					}
-					writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse)
-
-					if r.progress != nil {
-						r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
-					}
-
-					return writeErr
-				}
-				err := sanitizeError(file, writeToFile())
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		for file := range pack.files {
-			if errFile := sanitizeError(file, err); errFile != nil {
-				return errFile
-			}
-		}
-	}
-
-	return nil
+		})
 }

internal/restorer/filerestorer_test.go

@@ -247,6 +247,27 @@ func TestFileRestorerPackSkip(t *testing.T) {
 	}
 }
 
+func TestFileRestorerFrequentBlob(t *testing.T) {
+	tempdir := rtest.TempDir(t)
+
+	for _, sparse := range []bool{false, true} {
+		blobs := []TestBlob{
+			{"data1-1", "pack1-1"},
+		}
+		for i := 0; i < 10000; i++ {
+			blobs = append(blobs, TestBlob{"a", "pack1-1"})
+		}
+		blobs = append(blobs, TestBlob{"end", "pack1-1"})
+
+		restoreAndVerify(t, tempdir, []TestFile{
+			{
+				name:  "file1",
+				blobs: blobs,
+			},
+		}, nil, sparse)
+	}
+}
+
 func TestErrorRestoreFiles(t *testing.T) {
 	tempdir := rtest.TempDir(t)
 	content := []TestFile{
@@ -316,3 +337,47 @@ func testPartialDownloadError(t *testing.T, part int) {
 	rtest.OK(t, err)
 	verifyRestore(t, r, repo)
 }
+
+func TestFatalDownloadError(t *testing.T) {
+	tempdir := rtest.TempDir(t)
+	content := []TestFile{
+		{
+			name: "file1",
+			blobs: []TestBlob{
+				{"data1-1", "pack1"},
+				{"data1-2", "pack1"},
+			},
+		},
+		{
+			name: "file2",
+			blobs: []TestBlob{
+				{"data2-1", "pack1"},
+				{"data2-2", "pack1"},
+				{"data2-3", "pack1"},
+			},
+		}}
+
+	repo := newTestRepo(content)
+
+	loader := repo.loader
+	repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+		// only return half the data to break file2
+		return loader(ctx, h, length/2, offset, fn)
+	}
+
+	r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil)
+	r.files = repo.files
+
+	var errors []string
+	r.Error = func(s string, e error) error {
+		// ignore errors as in the `restore` command
+		errors = append(errors, s)
+		return nil
+	}
+
+	err := r.restoreFiles(context.TODO())
+	rtest.OK(t, err)
+
+	rtest.Assert(t, len(errors) == 1, "unexpected number of restore errors, expected: 1, got: %v", len(errors))
+	rtest.Assert(t, errors[0] == "file2", "expected error for file2, got: %v", errors[0])
+}