Add version for 0.16.3

rclone returns a "not found" error if an internal error occurs while
listing a folder. Ignoring this error lets restic erroneously think that
there are no files, which can cause `prune` to wipe the whole
repository.

.github/workflows/docker.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@b4bedf8053341df3b5a9f9e0f2cf4e79e27360c6
+        uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}

.github/workflows/tests.yml

@@ -62,7 +62,7 @@ jobs:
 
     steps:
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -226,7 +226,7 @@ jobs:
 
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
@@ -244,7 +244,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
@@ -255,7 +255,7 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.52.2
+          version: v1.55.2
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get

.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: doc/conf.py
+
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: doc/requirements.txt

VERSION

@@ -1 +1 @@
-0.16.1
+0.16.3

changelog/0.14.0_2022-08-25/pull-3475

@@ -3,7 +3,7 @@ Enhancement: Allow limiting IO concurrency for local and SFTP backend
 Restic did not support limiting the IO concurrency / number of connections for
 accessing repositories stored using the local or SFTP backends. The number of
 connections is now limited as for other backends, and can be configured via the
-the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures
-that restic does not overwhelm the backend with concurrent IO operations.
+`-o local.connections=2` and `-o sftp.connections=5` options. This ensures that
+restic does not overwhelm the backend with concurrent IO operations.
 
 https://github.com/restic/restic/pull/3475

changelog/0.16.2_2023-10-29/issue-4540

@@ -0,0 +1,9 @@
+Bugfix: Restore ARMv5 support for ARM binaries
+
+The official release binaries for restic 0.16.1 were accidentally built to
+require ARMv7. The build process is now updated to restore support for ARMv5.
+
+Please note that restic 0.17.0 will drop support for ARMv5 and require at least
+ARMv6.
+
+https://github.com/restic/restic/issues/4540

changelog/0.16.2_2023-10-29/pull-4545

@@ -0,0 +1,8 @@
+Bugfix: Repair documentation build on Read the Docs
+
+For restic 0.16.1, no documentation was available at
+https://restic.readthedocs.io/ .
+
+The documentation build process is now updated to work again.
+
+https://github.com/restic/restic/pull/4545

changelog/0.16.3_2024-01-14/issue-4560

@@ -0,0 +1,14 @@
+Bugfix: Improve errors for irregular files on Windows
+
+Since Go 1.21, most filesystem reparse points on Windows are considered to be
+irregular files. This caused restic to show an `error: invalid node type ""`
+error message for those files.
+
+This error message has now been improved and includes the relevant file path:
+`error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`.
+As irregular files are not required to behave like regular files, it is not
+possible to provide a generic way to back up those files.
+
+https://github.com/restic/restic/issues/4560
+https://github.com/restic/restic/pull/4620
+https://forum.restic.net/t/windows-backup-error-invalid-node-type/6875

changelog/0.16.3_2024-01-14/issue-4574

@@ -0,0 +1,11 @@
+Bugfix: Support backup of deduplicated files on Windows again
+
+With the official release builds of restic 0.16.1 and 0.16.2, it was not
+possible to back up files that were deduplicated by the corresponding
+Windows Server feature. This also applied to restic versions built using
+Go 1.21.0-1.21.4.
+
+The Go version used to build restic has now been updated to fix this.
+
+https://github.com/restic/restic/issues/4574
+https://github.com/restic/restic/pull/4621

changelog/0.16.3_2024-01-14/issue-4612

@@ -0,0 +1,11 @@
+Bugfix: Improve error handling for `rclone` backend
+
+Since restic 0.16.0, if rclone encountered an error while listing files,
+this could in rare circumstances cause restic to assume that there are no
+files. Although unlikely, this situation could result in data loss if it
+were to happen right when the `prune` command is listing existing snapshots.
+
+Error handling has now been improved to detect and work around this case.
+
+https://github.com/restic/restic/issues/4612
+https://github.com/restic/restic/pull/4618

changelog/0.16.3_2024-01-14/pull-4624

@@ -0,0 +1,11 @@
+Bugfix: Correct `restore` progress information if an error occurs
+
+If an error occurred while restoring a snapshot, this could cause the `restore`
+progress bar to show incorrect information. In addition, if a data file could
+not be loaded completely, then errors would also be reported for some already
+restored files.
+
+Error reporting of the `restore` command has now been made more accurate.
+
+https://github.com/restic/restic/pull/4624
+https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943

changelog/0.16.3_2024-01-14/pull-4626

@@ -0,0 +1,11 @@
+Bugfix: Improve reliability of restoring large files
+
+In some cases restic failed to restore large files that frequently contain the
+same file chunk. In combination with certain backends, this could result in
+network connection timeouts that caused incomplete restores.
+
+Restic now includes special handling for such file chunks to ensure reliable
+restores.
+
+https://github.com/restic/restic/pull/4626
+https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943

changelog/CHANGELOG.tmpl

@@ -1,18 +1,21 @@
-{{- range $changes := . }}{{ with $changes -}}
-Changelog for restic {{ .Version }} ({{ .Date }})
-=======================================
+# Table of Contents
 
+{{ range . -}}
+  * [Changelog for {{ .Version }}](#changelog-for-restic-{{ .Version | replace "." ""}}-{{ .Date | lower -}})
+{{ end -}}
+
+{{- range $changes := . }}{{ with $changes }}
+
+# Changelog for restic {{ .Version }} ({{ .Date }})
 The following sections list the changes in restic {{ .Version }} relevant to
 restic users. The changes are ordered by importance.
 
-Summary
--------
+## Summary
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .TypeShort }} #{{ .PrimaryID }}: {{ .Title }}
 {{- end }}{{ end }}
 
-Details
--------
+## Details
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }}
 {{ range $par := .Paragraphs }}
@@ -27,6 +30,5 @@ Details
 {{ range $url := .OtherURLs }}
    {{ $url -}}
 {{ end }}
-{{ end }}{{ end }}
-
+{{ end }}{{ end -}}
 {{ end }}{{ end -}}

cmd/restic/cmd_init.go

@@ -75,7 +75,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	repo, err := ReadRepo(gopts)
+	gopts.Repo, err = ReadRepo(gopts)
 	if err != nil {
 		return err
 	}
@@ -87,7 +87,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	be, err := create(ctx, repo, gopts, gopts.extended)
+	be, err := create(ctx, gopts.Repo, gopts, gopts.extended)
 	if err != nil {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
 	}

cmd/restic/global.go

@@ -43,7 +43,7 @@ import (
 	"golang.org/x/term"
 )
 
-var version = "0.16.1"
+var version = "0.16.3"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"

cmd/restic/integration_helpers_test.go

@@ -123,9 +123,8 @@ func directoriesContentsDiff(dir1, dir2 string) string {
 				fmt.Fprintf(&out, "+%v\n", b.path)
 				b = nil
 				continue
-			} else {
-				fmt.Fprintf(&out, "%%%v\n", a.path)
 			}
+			fmt.Fprintf(&out, "%%%v\n", a.path)
 		}
 
 		a, b = nil, nil

doc/020_installation.rst

@@ -84,6 +84,12 @@ If you are using macOS, you can install restic using the
 
     $ brew install restic
 
+On Linux and macOS, you can also install it using `pkgx <https://pkgx.sh/>`__:
+
+.. code-block:: console
+
+    $ pkgx install restic
+
 You may also install it using `MacPorts <https://www.macports.org/>`__:
 
 .. code-block:: console

doc/030_preparing_a_new_repo.rst

@@ -331,7 +331,7 @@ Wasabi
 ************
 
 `Wasabi <https://wasabi.com>`__ is a low cost Amazon S3 conformant object storage provider.
-Due to it's S3 conformance, Wasabi can be used as a storage provider for a restic repository.
+Due to its S3 conformance, Wasabi can be used as a storage provider for a restic repository.
 
 -  Create a Wasabi bucket using the `Wasabi Console <https://console.wasabisys.com>`__.
 -  Determine the correct Wasabi service URL for your bucket `here <https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031-What-are-the-service-URLs-for-Wasabi-s-different-regions->`__.
@@ -822,7 +822,7 @@ To make this work we can employ the help of the ``setgid`` permission bit
 available on Linux and most other Unix systems. This permission bit makes
 newly created directories inherit both the group owner (gid) and setgid bit
 from the parent directory. Setting this bit requires root but since it
-propagates down to any new directories we only have to do this priviledged
+propagates down to any new directories we only have to do this privileged
 setup once:
 
 .. code-block:: console

doc/040_backup.rst

@@ -146,7 +146,7 @@ change detection rule based on file metadata to determine whether a file is
 likely unchanged since a previous backup. If it is, the file is not scanned
 again.
 
-The previous backup snapshot, called "parent" snaphot in restic terminology,
+The previous backup snapshot, called "parent" snapshot in restic terminology,
 is determined as follows. By default restic groups snapshots by hostname and
 backup paths, and then selects the latest snapshot in the group that matches
 the current backup. You can change the selection criteria using the

doc/045_working_with_repos.rst

@@ -120,7 +120,7 @@ be skipped by later copy runs.
 The source repository is specified with ``--from-repo`` or can be read
 from a file specified via ``--from-repository-file``. Both of these options
 can also be set as environment variables ``$RESTIC_FROM_REPOSITORY`` or
-``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the destination repository
+``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the source repository
 the password can be read from a file ``--from-password-file`` or from a command
 ``--from-password-command``.
 Alternatively the environment variables ``$RESTIC_FROM_PASSWORD_COMMAND`` and
@@ -337,7 +337,7 @@ over 5 separate invocations:
     $ restic -r /srv/restic-repo check --read-data-subset=4/5
     $ restic -r /srv/restic-repo check --read-data-subset=5/5
 
-Use ``--read-data-subset=x%`` to check a randomly choosen subset of the
+Use ``--read-data-subset=x%`` to check a randomly chosen subset of the
 repository pack files. It takes one parameter, ``x``, the percentage of
 pack files to check as an integer or floating point number. This will not
 guarantee to cover all available pack files after sufficient runs, but it is

doc/050_restore.rst

@@ -106,7 +106,7 @@ Restic supports storage and preservation of hard links. However, since
 hard links exist in the scope of a filesystem by definition, restoring
 hard links from a fuse mount should be done by a program that preserves
 hard links. A program that does so is ``rsync``, used with the option
---hard-links.
+``--hard-links``.
 
 .. note:: ``restic mount`` is mostly useful if you want to restore just a few
    files out of a snapshot, or to check which files are contained in a snapshot.

doc/060_forget.rst

@@ -207,10 +207,13 @@ The ``forget`` command accepts the following policy options:
     They also only count hours/days/weeks/etc which have one or more snapshots.
     A value of ``-1`` will be interpreted as "forever", i.e. "keep all".
 
-.. note:: All duration related options (``--keep-{within,-*}``) ignore snapshots
+.. note:: All duration related options (``--keep-{within-,}*``) ignore snapshots
     with a timestamp in the future (relative to when the ``forget`` command is
     run) and these snapshots will hence not be removed.
 
+.. note:: If there are not enough snapshots to keep one for each duration related
+    ``--keep-{within-,}*`` option, the oldest snapshot is kept additionally.
+
 .. note:: Specifying ``--keep-tag ''`` will match untagged snapshots only.
 
 When ``forget`` is run with a policy, restic first loads the list of all snapshots

doc/075_scripting.rst

@@ -556,7 +556,7 @@ The snapshots command returns a single JSON object, an array with objects of the
 stats
 -----
 
-The snapshots command returns a single JSON object.
+The stats command returns a single JSON object.
 
 +------------------------------+-----------------------------------------------------+
 | ``total_size``               | Repository size in bytes                            |

doc/conf.py

@@ -12,14 +12,16 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
-import os
 
 # -- General configuration ------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.extlinks']
+extensions = [
+    'sphinx.ext.extlinks',
+    'sphinx_rtd_theme',
+]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -35,7 +37,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'restic'
-copyright = '2018, restic authors'
+copyright = '2023, restic authors'
 author = 'fd0'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -54,7 +56,7 @@ release = version
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = "en"
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -72,21 +74,11 @@ todo_include_todos = False
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-#
-if os.environ.get('READTHEDOCS') == 'True':
-    html_context = {
-        'css_files': [
-            'https://media.readthedocs.org/css/sphinx_rtd_theme.css',
-            'https://media.readthedocs.org/css/readthedocs-doc-embed.css',
-            '_static/css/restic.css',
-        ]
-    }
-else:
-    # we're not built by rtd => add rtd-theme
-    import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-    html_style = 'css/restic.css'
+html_theme = 'sphinx_rtd_theme'
+
+html_css_files = [
+    'css/restic.css',
+]
 
 html_logo = 'logo/logo.png'
 

doc/manual_rest.rst

@@ -148,11 +148,11 @@ command:
       -v, --verbose                    be verbose (specify multiple times or a level using --verbose=n, max level/times is 2)
 
 Subcommands that support showing progress information such as ``backup``,
-``check`` and ``prune`` will do so unless the quiet flag ``-q`` or
-``--quiet`` is set. When running from a non-interactive console progress
-reporting is disabled by default to not fill your logs. For interactive
-and non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS``
-can be used to control the frequency of progress reporting. Use for example
+``restore``, ``check`` and ``prune`` will do so unless the quiet flag ``-q``
+or ``--quiet`` is set. When running from a non-interactive console progress
+reporting is disabled by default to not fill your logs. For interactive and
+non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS`` can
+be used to control the frequency of progress reporting. Use for example
 ``0.016666`` to only update the progress once per minute.
 
 Additionally, on Unix systems if ``restic`` receives a SIGUSR1 signal the

go.mod

@@ -1,8 +1,8 @@
 module github.com/restic/restic
 
 require (
-	cloud.google.com/go/storage v1.33.0
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0
+	cloud.google.com/go/storage v1.34.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
 	github.com/Backblaze/blazer v0.6.1
@@ -13,8 +13,8 @@ require (
 	github.com/go-ole/go-ole v1.3.0
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/klauspost/compress v1.17.2
-	github.com/minio/minio-go/v7 v7.0.63
+	github.com/klauspost/compress v1.17.4
+	github.com/minio/minio-go/v7 v7.0.66
 	github.com/minio/sha256-simd v1.0.1
 	github.com/ncw/swift/v2 v2.0.2
 	github.com/pkg/errors v0.9.1
@@ -25,15 +25,15 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	go.uber.org/automaxprocs v1.5.3
-	golang.org/x/crypto v0.14.0
-	golang.org/x/net v0.17.0
-	golang.org/x/oauth2 v0.13.0
-	golang.org/x/sync v0.4.0
-	golang.org/x/sys v0.13.0
-	golang.org/x/term v0.13.0
-	golang.org/x/text v0.13.0
-	golang.org/x/time v0.3.0
-	google.golang.org/api v0.148.0
+	golang.org/x/crypto v0.17.0
+	golang.org/x/net v0.19.0
+	golang.org/x/oauth2 v0.15.0
+	golang.org/x/sync v0.5.0
+	golang.org/x/sys v0.15.0
+	golang.org/x/term v0.15.0
+	golang.org/x/text v0.14.0
+	golang.org/x/time v0.5.0
+	google.golang.org/api v0.149.0
 )
 
 require (
@@ -41,7 +41,7 @@ require (
 	cloud.google.com/go/compute v1.23.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.3 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -51,12 +51,12 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/google/uuid v1.3.1 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect

go.sum

@@ -7,14 +7,14 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc=
 cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE=
-cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
-cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw=
+cloud.google.com/go/storage v1.34.0 h1:9KHBBTbaHPsNxO043SFmH3pMojjZiW+BFl9H41L7xjk=
+cloud.google.com/go/storage v1.34.0/go.mod h1:Eji+S0CCQebjsiXxyIvPItC3BN3zWsdJjWfHfoLblgY=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 h1:TuEMD+E+1aTjjLICGQOW6vLe8UWES7kopac9mUXL56Y=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
@@ -95,10 +95,10 @@ github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0Z
 github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
 github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ=
-github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -108,11 +108,11 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
-github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -122,8 +122,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
-github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
+github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
+github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -183,8 +183,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -202,18 +202,18 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
+golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
-golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -229,22 +229,22 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
-golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -258,8 +258,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
-google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs=
-google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU=
+google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY=
+google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=

helpers/build-release-binaries/main.go

@@ -125,6 +125,9 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) {
 		"GOOS="+goos,
 		"GOARCH="+goarch,
 	)
+	if goarch == "arm" {
+		c.Env = append(c.Env, "GOARM=5")
+	}
 	verbose("run %v %v in %v", "go", c.Args, c.Dir)
 
 	err := c.Run()

helpers/verify-release-binaries.sh

@@ -12,6 +12,10 @@ go_version="$2"
 
 # invalid if zero
 is_valid=1
+set_invalid() {
+    echo $1
+    is_valid=0
+}
 
 tmpdir="$(mktemp -d -p .)"
 cd "${tmpdir}"
@@ -41,7 +45,7 @@ for i in $(cat SHA256SUMS | cut -d " "  -f 3 ) ; do
     echo "Downloading $i"
     curl -OLSs https://github.com/restic/restic/releases/download/v${restic_version}/"$i"
 done
-shasum -a256 -c SHA256SUMS || echo "WARNING: RELEASE BINARIES DO NOT MATCH SHA256SUMS!" && is_valid=0
+shasum -a256 -c SHA256SUMS || set_invalid "WARNING: RELEASE BINARIES DO NOT MATCH SHA256SUMS!"
 gpg --verify restic-${restic_version}.tar.gz.asc restic-${restic_version}.tar.gz
 # TODO verify that the release does not contain any unexpected files
 
@@ -74,9 +78,9 @@ cp "restic-${restic_version}.tar.gz" output
 cp SHA256SUMS output
 
 # check that all release binaries have been reproduced successfully
-(cd output && shasum -a256 -c SHA256SUMS) || echo "WARNING: REPRODUCED BINARIES DO NOT MATCH RELEASE BINARIES!" && is_valid=0
+(cd output && shasum -a256 -c SHA256SUMS) || set_invalid "WARNING: REPRODUCED BINARIES DO NOT MATCH RELEASE BINARIES!"
 # and that the SHA256SUMS files does not miss binaries
-for i in output/restic* ; do grep "$(basename "$i")" SHA256SUMS > /dev/null || echo "WARNING: $i MISSING FROM RELEASE SHA256SUMS FILE!" && is_valid=0 ; done
+for i in output/restic* ; do grep "$(basename "$i")" SHA256SUMS > /dev/null || set_invalid "WARNING: $i MISSING FROM RELEASE SHA256SUMS FILE!" ; done
 
 
 extract_docker() {
@@ -95,8 +99,7 @@ extract_docker() {
         tar -xvf "$i" -C img usr/bin/restic 2> /dev/null 1>&2 || true
         if [[ -f img/usr/bin/restic ]]; then
             if [[ -f restic-docker ]]; then
-                echo "WARNING: CONTAINER CONTAINS MULTIPLE RESTIC BINARIES"
-                is_valid=0
+                set_invalid "WARNING: CONTAINER CONTAINS MULTIPLE RESTIC BINARIES"
             fi
             mv img/usr/bin/restic restic-docker
         fi
@@ -118,7 +121,7 @@ for img in restic/restic ghcr.io/restic/restic; do
     extract_docker "$img" 386 386
     extract_docker "$img" amd64 amd64
 
-    (cd docker && shasum -a256 -c SHA256SUMS) || echo "WARNING: DOCKER CONTAINER DOES NOT CONTAIN RELEASE BINARIES!" && is_valid=0
+    (cd docker && shasum -a256 -c SHA256SUMS) || set_invalid "WARNING: DOCKER CONTAINER DOES NOT CONTAIN RELEASE BINARIES!"
 
     mv docker docker-$(( ctr++ ))
 done

internal/archiver/archiver.go

@@ -2,10 +2,12 @@ package archiver
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path"
 	"runtime"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/debug"
@@ -168,6 +170,11 @@ func (arch *Archiver) error(item string, err error) error {
 		return err
 	}
 
+	// not all errors include the filepath, thus add it if it is missing
+	if !strings.Contains(err.Error(), item) {
+		err = fmt.Errorf("%v: %w", item, err)
+	}
+
 	errf := arch.Error(item, err)
 	if err != errf {
 		debug.Log("item %v: error was filtered by handler, before: %q, after: %v", item, err, errf)
@@ -183,7 +190,10 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo)
 	}
 	// overwrite name to match that within the snapshot
 	node.Name = path.Base(snPath)
-	return node, errors.WithStack(err)
+	if err != nil {
+		return node, fmt.Errorf("nodeFromFileInfo %v: %w", filename, err)
+	}
+	return node, err
 }
 
 // loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned.

internal/backend/rest/rest.go

@@ -328,8 +328,13 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
 	}
 
 	if resp.StatusCode == http.StatusNotFound {
-		// ignore missing directories
-		return nil
+		if !strings.HasPrefix(resp.Header.Get("Server"), "rclone/") {
+			// ignore missing directories, unless the server is rclone. rclone
+			// already ignores missing directories, but misuses "not found" to
+			// report certain internal errors, see
+			// https://github.com/rclone/rclone/pull/7550 for details.
+			return nil
+		}
 	}
 
 	if resp.StatusCode != http.StatusOK {

internal/fuse/file.go

@@ -143,7 +143,7 @@ func (f *openFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.R
 	// the methods being called are responsible for appropriate synchronization.
 	//
 	// However, no lock needed here as getBlobAt can be called conurrently
-	// (blobCache has it's own locking)
+	// (blobCache has its own locking)
 	for i := startContent; remainingBytes > 0 && i < len(f.cumsize)-1; i++ {
 		blob, err := f.getBlobAt(ctx, i)
 		if err != nil {

internal/fuse/snapshots_dir.go

@@ -110,9 +110,8 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
 			return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot)
 		} else if entry.snapshot != nil {
 			return newDirFromSnapshot(d.root, inode, entry.snapshot)
-		} else {
-			return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
 		}
+		return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
 	}
 
 	return nil, syscall.ENOENT

internal/repository/repository.go

@@ -829,7 +829,7 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic
 }
 
 // ListPack returns the list of blobs saved in the pack id and the length of
-// the the pack header.
+// the pack header.
 func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) {
 	h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 
@@ -887,9 +887,9 @@ type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset
 const maxUnusedRange = 4 * 1024 * 1024
 
 // StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to
-// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. In
-// case of download errors handleBlobFn might be called multiple times for the same blob. If the
-// callback returns an error, then StreamPack will abort and not retry it.
+// the handleBlobFn callback or an error if decryption failed or the blob hash does not match.
+// handleBlobFn is never called multiple times for the same blob. If the callback returns an error,
+// then StreamPack will abort and not retry it.
 func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error {
 	if len(blobs) == 0 {
 		// nothing to do
@@ -951,7 +951,9 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key,
 		currentBlobEnd := dataStart
 		var buf []byte
 		var decode []byte
-		for _, entry := range blobs {
+		for len(blobs) > 0 {
+			entry := blobs[0]
+
 			skipBytes := int(entry.Offset - currentBlobEnd)
 			if skipBytes < 0 {
 				return errors.Errorf("overlapping blobs in pack %v", packID)
@@ -1014,6 +1016,8 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key,
 				cancel()
 				return backoff.Permanent(err)
 			}
+			// ensure that each blob is only passed once to handleBlobFn
+			blobs = blobs[1:]
 		}
 		return nil
 	})

internal/repository/repository_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -14,6 +15,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/google/go-cmp/cmp"
 	"github.com/klauspost/compress/zstd"
 	"github.com/restic/restic/internal/backend/local"
@@ -528,7 +530,9 @@ func testStreamPack(t *testing.T, version uint) {
 	packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress)
 
 	loadCalls := 0
-	load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+	shortFirstLoad := false
+
+	loadBytes := func(length int, offset int64) []byte {
 		data := packfile
 
 		if offset > int64(len(data)) {
@@ -540,32 +544,56 @@ func testStreamPack(t *testing.T, version uint) {
 		if length > len(data) {
 			length = len(data)
 		}
+		if shortFirstLoad {
+			length /= 2
+			shortFirstLoad = false
+		}
+
+		return data[:length]
+	}
+
+	load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+		data := loadBytes(length, offset)
+		if shortFirstLoad {
+			data = data[:len(data)/2]
+			shortFirstLoad = false
+		}
 
-		data = data[:length]
 		loadCalls++
 
-		return fn(bytes.NewReader(data))
+		err := fn(bytes.NewReader(data))
+		if err == nil {
+			return nil
+		}
+		var permanent *backoff.PermanentError
+		if errors.As(err, &permanent) {
+			return err
+		}
 
+		// retry loading once
+		return fn(bytes.NewReader(loadBytes(length, offset)))
 	}
 
 	// first, test regular usage
 	t.Run("regular", func(t *testing.T) {
 		tests := []struct {
-			blobs []restic.Blob
-			calls int
+			blobs          []restic.Blob
+			calls          int
+			shortFirstLoad bool
 		}{
-			{packfileBlobs[1:2], 1},
-			{packfileBlobs[2:5], 1},
-			{packfileBlobs[2:8], 1},
+			{packfileBlobs[1:2], 1, false},
+			{packfileBlobs[2:5], 1, false},
+			{packfileBlobs[2:8], 1, false},
 			{[]restic.Blob{
 				packfileBlobs[0],
 				packfileBlobs[4],
 				packfileBlobs[2],
-			}, 1},
+			}, 1, false},
 			{[]restic.Blob{
 				packfileBlobs[0],
 				packfileBlobs[len(packfileBlobs)-1],
-			}, 2},
+			}, 2, false},
+			{packfileBlobs[:], 1, true},
 		}
 
 		for _, test := range tests {
@@ -592,6 +620,7 @@ func testStreamPack(t *testing.T, version uint) {
 				}
 
 				loadCalls = 0
+				shortFirstLoad = test.shortFirstLoad
 				err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob)
 				if err != nil {
 					t.Fatal(err)
@@ -604,6 +633,7 @@ func testStreamPack(t *testing.T, version uint) {
 			})
 		}
 	})
+	shortFirstLoad = false
 
 	// next, test invalid uses, which should return an error
 	t.Run("invalid", func(t *testing.T) {

internal/restic/node.go

@@ -109,7 +109,7 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
 }
 
 func nodeTypeFromFileInfo(fi os.FileInfo) string {
-	switch fi.Mode() & (os.ModeType | os.ModeCharDevice) {
+	switch fi.Mode() & os.ModeType {
 	case 0:
 		return "file"
 	case os.ModeDir:
@@ -124,6 +124,8 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
 		return "fifo"
 	case os.ModeSocket:
 		return "socket"
+	case os.ModeIrregular:
+		return "irregular"
 	}
 
 	return ""
@@ -622,7 +624,7 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
 	case "fifo":
 	case "socket":
 	default:
-		return errors.Errorf("invalid node type %q", node.Type)
+		return errors.Errorf("unsupported file type %q", node.Type)
 	}
 
 	return node.fillExtendedAttributes(path)

internal/restic/repository.go

@@ -39,7 +39,7 @@ type Repository interface {
 	List(ctx context.Context, t FileType, fn func(ID, int64) error) error
 
 	// ListPack returns the list of blobs saved in the pack id and the length of
-	// the the pack header.
+	// the pack header.
 	ListPack(context.Context, ID, int64) ([]Blob, uint32, error)
 
 	LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error)

internal/restorer/filerestorer.go

@@ -197,19 +197,20 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
 	return wg.Wait()
 }
 
-func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
+type blobToFileOffsetsMapping map[restic.ID]struct {
+	files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file
+	blob  restic.Blob
+}
 
+func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
 	// calculate blob->[]files->[]offsets mappings
-	blobs := make(map[restic.ID]struct {
-		files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file
-	})
-	var blobList []restic.Blob
+	blobs := make(blobToFileOffsetsMapping)
 	for file := range pack.files {
 		addBlob := func(blob restic.Blob, fileOffset int64) {
 			blobInfo, ok := blobs[blob.ID]
 			if !ok {
 				blobInfo.files = make(map[*fileInfo][]int64)
-				blobList = append(blobList, blob)
+				blobInfo.blob = blob
 				blobs[blob.ID] = blobInfo
 			}
 			blobInfo.files[file] = append(blobInfo.files[file], fileOffset)
@@ -239,65 +240,120 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
 		}
 	}
 
-	sanitizeError := func(file *fileInfo, err error) error {
-		if err != nil {
-			err = r.Error(file.location, err)
+	// track already processed blobs for precise error reporting
+	processedBlobs := restic.NewBlobSet()
+	for _, entry := range blobs {
+		occurrences := 0
+		for _, offsets := range entry.files {
+			occurrences += len(offsets)
+		}
+		// With a maximum blob size of 8MB, the normal blob streaming has to write
+		// at most 800MB for a single blob. This should be short enough to avoid
+		// network connection timeouts. Based on a quick test, a limit of 100 only
+		// selects a very small number of blobs (the number of references per blob
+		// - aka. `count` - seem to follow a expontential distribution)
+		if occurrences > 100 {
+			// process frequently referenced blobs first as these can take a long time to write
+			// which can cause backend connections to time out
+			delete(blobs, entry.blob.ID)
+			partialBlobs := blobToFileOffsetsMapping{entry.blob.ID: entry}
+			err := r.downloadBlobs(ctx, pack.id, partialBlobs, processedBlobs)
+			if err := r.reportError(blobs, processedBlobs, err); err != nil {
+				return err
+			}
 		}
-		return err
 	}
 
-	err := repository.StreamPack(ctx, r.packLoader, r.key, pack.id, blobList, func(h restic.BlobHandle, blobData []byte, err error) error {
-		blob := blobs[h.ID]
-		if err != nil {
-			for file := range blob.files {
-				if errFile := sanitizeError(file, err); errFile != nil {
-					return errFile
+	if len(blobs) == 0 {
+		return nil
+	}
+
+	err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs)
+	return r.reportError(blobs, processedBlobs, err)
+}
+
+func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error {
+	if err != nil {
+		err = r.Error(file.location, err)
+	}
+	return err
+}
+
+func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet, err error) error {
+	if err == nil {
+		return nil
+	}
+
+	// only report error for not yet processed blobs
+	affectedFiles := make(map[*fileInfo]struct{})
+	for _, entry := range blobs {
+		if processedBlobs.Has(entry.blob.BlobHandle) {
+			continue
+		}
+		for file := range entry.files {
+			affectedFiles[file] = struct{}{}
+		}
+	}
+
+	for file := range affectedFiles {
+		if errFile := r.sanitizeError(file, err); errFile != nil {
+			return errFile
+		}
+	}
+	return nil
+}
+
+func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID,
+	blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error {
+
+	blobList := make([]restic.Blob, 0, len(blobs))
+	for _, entry := range blobs {
+		blobList = append(blobList, entry.blob)
+	}
+	return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList,
+		func(h restic.BlobHandle, blobData []byte, err error) error {
+			processedBlobs.Insert(h)
+			blob := blobs[h.ID]
+			if err != nil {
+				for file := range blob.files {
+					if errFile := r.sanitizeError(file, err); errFile != nil {
+						return errFile
+					}
+				}
+				return nil
+			}
+			for file, offsets := range blob.files {
+				for _, offset := range offsets {
+					writeToFile := func() error {
+						// this looks overly complicated and needs explanation
+						// two competing requirements:
+						// - must create the file once and only once
+						// - should allow concurrent writes to the file
+						// so write the first blob while holding file lock
+						// write other blobs after releasing the lock
+						createSize := int64(-1)
+						file.lock.Lock()
+						if file.inProgress {
+							file.lock.Unlock()
+						} else {
+							defer file.lock.Unlock()
+							file.inProgress = true
+							createSize = file.size
+						}
+						writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse)
+
+						if r.progress != nil {
+							r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
+						}
+
+						return writeErr
+					}
+					err := r.sanitizeError(file, writeToFile())
+					if err != nil {
+						return err
+					}
 				}
 			}
 			return nil
-		}
-		for file, offsets := range blob.files {
-			for _, offset := range offsets {
-				writeToFile := func() error {
-					// this looks overly complicated and needs explanation
-					// two competing requirements:
-					// - must create the file once and only once
-					// - should allow concurrent writes to the file
-					// so write the first blob while holding file lock
-					// write other blobs after releasing the lock
-					createSize := int64(-1)
-					file.lock.Lock()
-					if file.inProgress {
-						file.lock.Unlock()
-					} else {
-						defer file.lock.Unlock()
-						file.inProgress = true
-						createSize = file.size
-					}
-					writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse)
-
-					if r.progress != nil {
-						r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
-					}
-
-					return writeErr
-				}
-				err := sanitizeError(file, writeToFile())
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		for file := range pack.files {
-			if errFile := sanitizeError(file, err); errFile != nil {
-				return errFile
-			}
-		}
-	}
-
-	return nil
+		})
 }

internal/restorer/filerestorer_test.go

@@ -247,6 +247,27 @@ func TestFileRestorerPackSkip(t *testing.T) {
 	}
 }
 
+func TestFileRestorerFrequentBlob(t *testing.T) {
+	tempdir := rtest.TempDir(t)
+
+	for _, sparse := range []bool{false, true} {
+		blobs := []TestBlob{
+			{"data1-1", "pack1-1"},
+		}
+		for i := 0; i < 10000; i++ {
+			blobs = append(blobs, TestBlob{"a", "pack1-1"})
+		}
+		blobs = append(blobs, TestBlob{"end", "pack1-1"})
+
+		restoreAndVerify(t, tempdir, []TestFile{
+			{
+				name:  "file1",
+				blobs: blobs,
+			},
+		}, nil, sparse)
+	}
+}
+
 func TestErrorRestoreFiles(t *testing.T) {
 	tempdir := rtest.TempDir(t)
 	content := []TestFile{
@@ -316,3 +337,47 @@ func testPartialDownloadError(t *testing.T, part int) {
 	rtest.OK(t, err)
 	verifyRestore(t, r, repo)
 }
+
+func TestFatalDownloadError(t *testing.T) {
+	tempdir := rtest.TempDir(t)
+	content := []TestFile{
+		{
+			name: "file1",
+			blobs: []TestBlob{
+				{"data1-1", "pack1"},
+				{"data1-2", "pack1"},
+			},
+		},
+		{
+			name: "file2",
+			blobs: []TestBlob{
+				{"data2-1", "pack1"},
+				{"data2-2", "pack1"},
+				{"data2-3", "pack1"},
+			},
+		}}
+
+	repo := newTestRepo(content)
+
+	loader := repo.loader
+	repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+		// only return half the data to break file2
+		return loader(ctx, h, length/2, offset, fn)
+	}
+
+	r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil)
+	r.files = repo.files
+
+	var errors []string
+	r.Error = func(s string, e error) error {
+		// ignore errors as in the `restore` command
+		errors = append(errors, s)
+		return nil
+	}
+
+	err := r.restoreFiles(context.TODO())
+	rtest.OK(t, err)
+
+	rtest.Assert(t, len(errors) == 1, "unexpected number of restore errors, expected: 1, got: %v", len(errors))
+	rtest.Assert(t, errors[0] == "file2", "expected error for file2, got: %v", errors[0])
+}