Add version for 0.17.3

doc: Polish changelogs before release

.github/workflows/docker.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}

.github/workflows/tests.yml

@@ -66,6 +66,9 @@ jobs:
       GOPROXY: https://proxy.golang.org
 
     steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
       - name: Set up Go ${{ matrix.go }}
         uses: actions/setup-go@v5
         with:
@@ -139,9 +142,6 @@ jobs:
           echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH
         if: matrix.os == 'windows-latest'
 
-      - name: Check out code
-        uses: actions/checkout@v4
-
       - name: Build with build.go
         run: |
           go run build.go
@@ -230,14 +230,14 @@ jobs:
     name: Cross Compile for subset ${{ matrix.subset }}
 
     steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
       - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
-      - name: Check out code
-        uses: actions/checkout@v4
-
       - name: Cross-compile for subset ${{ matrix.subset }}
         run: |
           mkdir build-output build-output-debug
@@ -252,14 +252,14 @@ jobs:
       # allow annotating code in the PR
       checks: write
     steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
       - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v5
         with:
           go-version: ${{ env.latest_go }}
 
-      - name: Check out code
-        uses: actions/checkout@v4
-
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v6
         with:

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Table of Contents
 
+* [Changelog for 0.17.3](#changelog-for-restic-0173-2024-11-08)
+* [Changelog for 0.17.2](#changelog-for-restic-0172-2024-10-27)
+* [Changelog for 0.17.1](#changelog-for-restic-0171-2024-09-05)
 * [Changelog for 0.17.0](#changelog-for-restic-0170-2024-07-26)
 * [Changelog for 0.16.5](#changelog-for-restic-0165-2024-07-01)
 * [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04)
@@ -35,6 +38,384 @@
 * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
 
 
+# Changelog for restic 0.17.3 (2024-11-08)
+The following sections list the changes in restic 0.17.3 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #4971: Fix unusable `mount` on macOS Sonoma
+ * Fix #5003: Fix metadata errors during backup of removable disks on Windows
+ * Fix #5101: Do not retry load/list operation if SFTP connection is broken
+ * Fix #5107: Fix metadata error on Windows for backups using VSS
+ * Enh #5096: Allow `prune --dry-run` without lock
+
+## Details
+
+ * Bugfix #4971: Fix unusable `mount` on macOS Sonoma
+
+   On macOS Sonoma when using FUSE-T, it was not possible to access files in a
+   mounted repository. This issue is now resolved.
+
+   https://github.com/restic/restic/issues/4971
+   https://github.com/restic/restic/pull/5048
+
+ * Bugfix #5003: Fix metadata errors during backup of removable disks on Windows
+
+   Since restic 0.17.0, backing up removable disks on Windows could report errors
+   with retrieving metadata like shown below.
+
+   ```
+   error: incomplete metadata for d:\filename: get named security info failed with: Access is denied.
+   ```
+
+   This has now been fixed.
+
+   https://github.com/restic/restic/issues/5003
+   https://github.com/restic/restic/pull/5123
+   https://forum.restic.net/t/backing-up-a-folder-from-a-veracrypt-volume-brings-up-errors-since-restic-v17-0/8444
+
+ * Bugfix #5101: Do not retry load/list operation if SFTP connection is broken
+
+   When using restic with the SFTP backend, backend operations that load a file or
+   list files were retried even if the SFTP connection was broken. This has now
+   been fixed.
+
+   https://github.com/restic/restic/pull/5101
+   https://forum.restic.net/t/restic-hanging-on-backup/8559
+
+ * Bugfix #5107: Fix metadata error on Windows for backups using VSS
+
+   Since restic 0.17.2, when creating a backup on Windows using
+   `--use-fs-snapshot`, restic would report an error like the following:
+
+   ```
+   error: incomplete metadata for C:\: get EA failed while opening file handle for path \\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyXX\, with: The process cannot access the file because it is being used by another process.
+   ```
+
+   This has now been fixed by correctly handling paths that refer to volume shadow
+   copy snapshots.
+
+   https://github.com/restic/restic/issues/5107
+   https://github.com/restic/restic/pull/5110
+   https://github.com/restic/restic/pull/5112
+
+ * Enhancement #5096: Allow `prune --dry-run` without lock
+
+   The `prune --dry-run --no-lock` now allows performing a dry-run without locking
+   the repository. Note that if the repository is modified concurrently, `prune`
+   may return inaccurate statistics or errors.
+
+   https://github.com/restic/restic/pull/5096
+
+
+# Changelog for restic 0.17.2 (2024-10-27)
+The following sections list the changes in restic 0.17.2 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #4004: Support container-level SAS/SAT tokens for Azure backend
+ * Fix #5047: Resolve potential error during concurrent cache cleanup
+ * Fix #5050: Return error if `tag` fails to lock repository
+ * Fix #5057: Exclude irregular files from backups
+ * Fix #5063: Correctly `backup` extended metadata when using VSS on Windows
+
+## Details
+
+ * Bugfix #4004: Support container-level SAS/SAT tokens for Azure backend
+
+   Restic previously expected SAS/SAT tokens to be generated at the account level,
+   which prevented tokens created at the container level from being used to
+   initialize a repository. This caused an error when attempting to initialize a
+   repository with container-level tokens.
+
+   Restic now supports both account-level and container-level SAS/SAT tokens for
+   initializing a repository.
+
+   https://github.com/restic/restic/issues/4004
+   https://github.com/restic/restic/pull/5093
+
+ * Bugfix #5047: Resolve potential error during concurrent cache cleanup
+
+   When multiple restic processes ran concurrently, they could compete to remove
+   obsolete snapshots from the local backend cache, sometimes leading to a "no such
+   file or directory" error. Restic now suppresses this error to prevent issues
+   during cache cleanup.
+
+   https://github.com/restic/restic/pull/5047
+
+ * Bugfix #5050: Return error if `tag` fails to lock repository
+
+   Since restic 0.17.0, the `tag` command did not return an error when it failed to
+   open or lock the repository. This issue has now been fixed.
+
+   https://github.com/restic/restic/issues/5050
+   https://github.com/restic/restic/pull/5056
+
+ * Bugfix #5057: Exclude irregular files from backups
+
+   Since restic 0.17.1, files with the type `irregular` could mistakenly be
+   included in snapshots, especially when backing up special file types on Windows
+   that restic cannot process. This issue has now been fixed.
+
+   Previously, this bug caused the `check` command to report errors like the
+   following one:
+
+   ```
+     tree 12345678[...]: node "example.zip" with invalid type "irregular"
+   ```
+
+   To repair affected snapshots, upgrade to restic 0.17.2 and run:
+
+   ```
+   restic repair snapshots --forget
+   ```
+
+   This will remove the `irregular` files from the snapshots (creating a new
+   snapshot ID for each of the affected snapshots).
+
+   https://github.com/restic/restic/pull/5057
+   https://forum.restic.net/t/errors-found-by-check-1-invalid-type-irregular-2-ciphertext-verification-failed/8447/2
+
+ * Bugfix #5063: Correctly `backup` extended metadata when using VSS on Windows
+
+   On Windows, when creating a backup with the `--use-fs-snapshot` option, restic
+   read extended metadata from the original filesystem path instead of from the
+   snapshot. This could result in errors if files were removed during the backup
+   process.
+
+   This issue has now been resolved.
+
+   https://github.com/restic/restic/issues/5063
+   https://github.com/restic/restic/pull/5097
+   https://github.com/restic/restic/pull/5099
+
+
+# Changelog for restic 0.17.1 (2024-09-05)
+The following sections list the changes in restic 0.17.1 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #2004: Correctly handle volume names in `backup` command on Windows
+ * Fix #4945: Include missing backup error text with `--json`
+ * Fix #4953: Correctly handle long paths on older Windows versions
+ * Fix #4957: Fix delayed cancellation of certain commands
+ * Fix #4958: Don't ignore metadata-setting errors during restore
+ * Fix #4969: Correctly restore timestamp for files with resource forks on macOS
+ * Fix #4975: Prevent `backup --stdin-from-command` from panicking
+ * Fix #4980: Skip extended attribute processing on unsupported Windows volumes
+ * Fix #5004: Fix spurious "A Required Privilege Is Not Held by the Client" error
+ * Fix #5005: Fix rare failures to retry locking a repository
+ * Fix #5018: Improve HTTP/2 support for REST backend
+ * Chg #4953: Also back up files with incomplete metadata
+ * Enh #4795: Display progress bar for `restore --verify`
+ * Enh #4934: Automatically clear removed snapshots from cache
+ * Enh #4944: Print JSON-formatted errors during `restore --json`
+ * Enh #4959: Return exit code 12 for "bad password" errors
+ * Enh #4970: Make timeout for stuck requests customizable
+
+## Details
+
+ * Bugfix #2004: Correctly handle volume names in `backup` command on Windows
+
+   On Windows, when the specified backup target only included the volume name
+   without a trailing slash, for example, `C:`, then restoring the resulting
+   snapshot would result in an error. Note that using `C:\` as backup target worked
+   correctly.
+
+   Specifying volume names is now handled correctly. To restore snapshots created
+   before this bugfix, use the <snapshot>:<subpath> syntax. For example, to restore
+   a snapshot with ID `12345678` that backed up `C:`, use the following command:
+
+   ```
+   restic restore 12345678:/C/C:./ --target output/folder
+   ```
+
+   https://github.com/restic/restic/issues/2004
+   https://github.com/restic/restic/pull/5028
+
+ * Bugfix #4945: Include missing backup error text with `--json`
+
+   Previously, when running a backup with the `--json` option, restic failed to
+   include the actual error message in the output, resulting in `"error": {}` being
+   displayed.
+
+   This has now been fixed, and restic now includes the error text in JSON output.
+
+   https://github.com/restic/restic/issues/4945
+   https://github.com/restic/restic/pull/4946
+
+ * Bugfix #4953: Correctly handle long paths on older Windows versions
+
+   On older Windows versions, like Windows Server 2012, restic 0.17.0 failed to
+   back up files with long paths. This problem has now been resolved.
+
+   https://github.com/restic/restic/issues/4953
+   https://github.com/restic/restic/pull/4954
+
+ * Bugfix #4957: Fix delayed cancellation of certain commands
+
+   Since restic 0.17.0, some commands did not immediately respond to cancellation
+   via Ctrl-C (SIGINT) and continued running for a short period. The most affected
+   commands were `diff`,`find`, `ls`, `stats` and `rewrite`. This is now resolved.
+
+   https://github.com/restic/restic/issues/4957
+   https://github.com/restic/restic/pull/4960
+
+ * Bugfix #4958: Don't ignore metadata-setting errors during restore
+
+   Previously, restic used to ignore errors when setting timestamps, attributes, or
+   file modes during a restore. It now reports those errors, except for permission
+   related errors when running without root privileges.
+
+   https://github.com/restic/restic/pull/4958
+
+ * Bugfix #4969: Correctly restore timestamp for files with resource forks on macOS
+
+   On macOS, timestamps were not restored for files with resource forks. This has
+   now been fixed.
+
+   https://github.com/restic/restic/issues/4969
+   https://github.com/restic/restic/pull/5006
+
+ * Bugfix #4975: Prevent `backup --stdin-from-command` from panicking
+
+   Restic would previously crash if `--stdin-from-command` was specified without
+   providing a command. This issue has now been fixed.
+
+   https://github.com/restic/restic/issues/4975
+   https://github.com/restic/restic/pull/4976
+
+ * Bugfix #4980: Skip extended attribute processing on unsupported Windows volumes
+
+   With restic 0.17.0, backups of certain Windows paths, such as network drives,
+   failed due to errors while fetching extended attributes.
+
+   Restic now skips extended attribute processing for volumes where they are not
+   supported.
+
+   https://github.com/restic/restic/issues/4955
+   https://github.com/restic/restic/issues/4950
+   https://github.com/restic/restic/pull/4980
+   https://github.com/restic/restic/pull/4998
+
+ * Bugfix #5004: Fix spurious "A Required Privilege Is Not Held by the Client" error
+
+   On Windows, creating a backup could sometimes trigger the following error:
+
+   ```
+   error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client.
+   ```
+
+   This has now been fixed.
+
+   https://github.com/restic/restic/issues/5004
+   https://github.com/restic/restic/pull/5019
+
+ * Bugfix #5005: Fix rare failures to retry locking a repository
+
+   Restic 0.17.0 could in rare cases fail to retry locking a repository if one of
+   the lock files failed to load, resulting in the error:
+
+   ```
+   unable to create lock in backend: circuit breaker open for file <lock/1234567890>
+   ```
+
+   This issue has now been addressed. The error handling now properly retries the
+   locking operation. In addition, restic waits a few seconds between locking
+   retries to increase chances of successful locking.
+
+   https://github.com/restic/restic/issues/5005
+   https://github.com/restic/restic/pull/5011
+   https://github.com/restic/restic/pull/5012
+
+ * Bugfix #5018: Improve HTTP/2 support for REST backend
+
+   If `rest-server` tried to gracefully shut down an HTTP/2 connection still in use
+   by the client, it could result in the following error:
+
+   ```
+   http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error
+   ```
+
+   This issue has now been resolved.
+
+   https://github.com/restic/restic/pull/5018
+   https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367
+
+ * Change #4953: Also back up files with incomplete metadata
+
+   If restic failed to read extended metadata for a file or folder during a backup,
+   then the file or folder was not included in the resulting snapshot. Instead, a
+   warning message was printed along with returning exit code 3 once the backup was
+   finished.
+
+   Now, restic also includes items for which the extended metadata could not be
+   read in a snapshot. The warning message has been updated to:
+
+   ```
+   incomplete metadata for /path/to/file: <details about error>
+   ```
+
+   https://github.com/restic/restic/issues/4953
+   https://github.com/restic/restic/pull/4977
+
+ * Enhancement #4795: Display progress bar for `restore --verify`
+
+   When the `restore` command is run with `--verify`, it now displays a progress
+   bar while the verification step is running. The progress bar is not shown when
+   the `--json` flag is specified.
+
+   https://github.com/restic/restic/issues/4795
+   https://github.com/restic/restic/pull/4989
+
+ * Enhancement #4934: Automatically clear removed snapshots from cache
+
+   Previously, restic only removed snapshots from the cache on the host where the
+   `forget` command was executed. On other hosts that use the same repository, the
+   old snapshots remained in the cache.
+
+   Restic now automatically clears old snapshots from the local cache of the
+   current host.
+
+   https://github.com/restic/restic/issues/4934
+   https://github.com/restic/restic/pull/4981
+
+ * Enhancement #4944: Print JSON-formatted errors during `restore --json`
+
+   Restic used to print any `restore` errors directly to the console as freeform
+   text messages, even when using the `--json` option.
+
+   Now, when `--json` is specified, restic prints them as JSON formatted messages.
+
+   https://github.com/restic/restic/issues/4944
+   https://github.com/restic/restic/pull/4946
+
+ * Enhancement #4959: Return exit code 12 for "bad password" errors
+
+   Restic now returns exit code 12 when it cannot open the repository due to an
+   incorrect password.
+
+   https://github.com/restic/restic/pull/4959
+
+ * Enhancement #4970: Make timeout for stuck requests customizable
+
+   Restic monitors connections to the backend to detect stuck requests. If a
+   request does not return any data within five minutes, restic assumes the request
+   is stuck and retries it. However, for large repositories this timeout might be
+   insufficient to collect a list of all files, causing the following error:
+
+   `List(data) returned error, retrying after 1s: [...]: request timeout`
+
+   It is now possible to increase the timeout using the `--stuck-request-timeout`
+   option.
+
+   https://github.com/restic/restic/issues/4970
+   https://github.com/restic/restic/pull/5014
+
+
 # Changelog for restic 0.17.0 (2024-07-26)
 The following sections list the changes in restic 0.17.0 relevant to
 restic users. The changes are ordered by importance.

VERSION

@@ -1 +1 @@
-0.17.0
+0.17.3

changelog/0.17.1_2024-09-05/issue-2004

@@ -0,0 +1,18 @@
+Bugfix: Correctly handle volume names in `backup` command on Windows
+
+On Windows, when the specified backup target only included the volume
+name without a trailing slash, for example, `C:`, then restoring the
+resulting snapshot would result in an error. Note that using `C:\`
+as backup target worked correctly.
+
+Specifying volume names is now handled correctly. To restore snapshots
+created before this bugfix, use the <snapshot>:<subpath> syntax. For
+example, to restore a snapshot with ID `12345678` that backed up `C:`,
+use the following command:
+
+```
+restic restore 12345678:/C/C:./ --target output/folder
+```
+
+https://github.com/restic/restic/issues/2004
+https://github.com/restic/restic/pull/5028

changelog/0.17.1_2024-09-05/issue-4795

@@ -0,0 +1,8 @@
+Enhancement: Display progress bar for `restore --verify`
+
+When the `restore` command is run with `--verify`, it now displays a progress
+bar while the verification step is running. The progress bar is not shown when
+the `--json` flag is specified.
+
+https://github.com/restic/restic/issues/4795
+https://github.com/restic/restic/pull/4989

changelog/0.17.1_2024-09-05/issue-4934

@@ -0,0 +1,11 @@
+Enhancement: Automatically clear removed snapshots from cache
+
+Previously, restic only removed snapshots from the cache on the host where the
+`forget` command was executed. On other hosts that use the same repository, the
+old snapshots remained in the cache.
+
+Restic now automatically clears old snapshots from the local cache of the
+current host.
+
+https://github.com/restic/restic/issues/4934
+https://github.com/restic/restic/pull/4981

changelog/0.17.1_2024-09-05/issue-4944

@@ -0,0 +1,9 @@
+Enhancement: Print JSON-formatted errors during `restore --json`
+
+Restic used to print any `restore` errors directly to the console as freeform
+text messages, even when using the `--json` option.
+
+Now, when `--json` is specified, restic prints them as JSON formatted messages.
+
+https://github.com/restic/restic/issues/4944
+https://github.com/restic/restic/pull/4946

changelog/0.17.1_2024-09-05/issue-4945

@@ -0,0 +1,10 @@
+Bugfix: Include missing backup error text with `--json`
+
+Previously, when running a backup with the `--json` option, restic failed to
+include the actual error message in the output, resulting in `"error": {}`
+being displayed.
+
+This has now been fixed, and restic now includes the error text in JSON output.
+
+https://github.com/restic/restic/issues/4945
+https://github.com/restic/restic/pull/4946

changelog/0.17.1_2024-09-05/issue-4953

@@ -0,0 +1,7 @@
+Bugfix: Correctly handle long paths on older Windows versions
+
+On older Windows versions, like Windows Server 2012, restic 0.17.0 failed to
+back up files with long paths. This problem has now been resolved.
+
+https://github.com/restic/restic/issues/4953
+https://github.com/restic/restic/pull/4954

changelog/0.17.1_2024-09-05/issue-4957

@@ -0,0 +1,8 @@
+Bugfix: Fix delayed cancellation of certain commands
+
+Since restic 0.17.0, some commands did not immediately respond to cancellation
+via Ctrl-C (SIGINT) and continued running for a short period. The most affected
+commands were `diff`,`find`, `ls`, `stats` and `rewrite`. This is now resolved.
+
+https://github.com/restic/restic/issues/4957
+https://github.com/restic/restic/pull/4960

changelog/0.17.1_2024-09-05/issue-4969

@@ -0,0 +1,7 @@
+Bugfix: Correctly restore timestamp for files with resource forks on macOS
+
+On macOS, timestamps were not restored for files with resource forks. This has
+now been fixed.
+
+https://github.com/restic/restic/issues/4969
+https://github.com/restic/restic/pull/5006

changelog/0.17.1_2024-09-05/issue-4970

@@ -0,0 +1,15 @@
+Enhancement: Make timeout for stuck requests customizable
+
+Restic monitors connections to the backend to detect stuck requests. If a
+request does not return any data within five minutes, restic assumes the
+request is stuck and retries it. However, for large repositories this timeout
+might be insufficient to collect a list of all files, causing the following
+error:
+
+`List(data) returned error, retrying after 1s: [...]: request timeout`
+
+It is now possible to increase the timeout using the `--stuck-request-timeout`
+option.
+
+https://github.com/restic/restic/issues/4970
+https://github.com/restic/restic/pull/5014

changelog/0.17.1_2024-09-05/issue-4975

@@ -0,0 +1,7 @@
+Bugfix: Prevent `backup --stdin-from-command` from panicking
+
+Restic would previously crash if `--stdin-from-command` was specified without
+providing a command. This issue has now been fixed.
+
+https://github.com/restic/restic/issues/4975
+https://github.com/restic/restic/pull/4976

changelog/0.17.1_2024-09-05/issue-5004

@@ -0,0 +1,12 @@
+Bugfix: Fix spurious "A Required Privilege Is Not Held by the Client" error
+
+On Windows, creating a backup could sometimes trigger the following error:
+
+```
+error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client.
+```
+
+This has now been fixed.
+
+https://github.com/restic/restic/issues/5004
+https://github.com/restic/restic/pull/5019

changelog/0.17.1_2024-09-05/issue-5005

@@ -0,0 +1,16 @@
+Bugfix: Fix rare failures to retry locking a repository
+
+Restic 0.17.0 could in rare cases fail to retry locking a repository if one of
+the lock files failed to load, resulting in the error:
+
+```
+unable to create lock in backend: circuit breaker open for file <lock/1234567890>
+```
+
+This issue has now been addressed. The error handling now properly retries the
+locking operation. In addition, restic waits a few seconds between locking
+retries to increase chances of successful locking.
+
+https://github.com/restic/restic/issues/5005
+https://github.com/restic/restic/pull/5011
+https://github.com/restic/restic/pull/5012

changelog/0.17.1_2024-09-05/pull-4958

@@ -0,0 +1,7 @@
+Bugfix: Don't ignore metadata-setting errors during restore
+
+Previously, restic used to ignore errors when setting timestamps, attributes,
+or file modes during a restore. It now reports those errors, except for
+permission related errors when running without root privileges.
+
+https://github.com/restic/restic/pull/4958

changelog/0.17.1_2024-09-05/pull-4959

@@ -0,0 +1,6 @@
+Enhancement: Return exit code 12 for "bad password" errors
+
+Restic now returns exit code 12 when it cannot open the repository due to an
+incorrect password.
+
+https://github.com/restic/restic/pull/4959

changelog/0.17.1_2024-09-05/pull-4977

@@ -0,0 +1,16 @@
+Change: Also back up files with incomplete metadata
+
+If restic failed to read extended metadata for a file or folder during a
+backup, then the file or folder was not included in the resulting snapshot.
+Instead, a warning message was printed along with returning exit code 3 once
+the backup was finished.
+
+Now, restic also includes items for which the extended metadata could not be
+read in a snapshot. The warning message has been updated to:
+
+```
+incomplete metadata for /path/to/file: <details about error>
+```
+
+https://github.com/restic/restic/issues/4953
+https://github.com/restic/restic/pull/4977

changelog/0.17.1_2024-09-05/pull-4980

@@ -0,0 +1,12 @@
+Bugfix: Skip extended attribute processing on unsupported Windows volumes
+
+With restic 0.17.0, backups of certain Windows paths, such as network drives,
+failed due to errors while fetching extended attributes.
+
+Restic now skips extended attribute processing for volumes where they are not
+supported.
+
+https://github.com/restic/restic/pull/4980
+https://github.com/restic/restic/pull/4998
+https://github.com/restic/restic/issues/4955
+https://github.com/restic/restic/issues/4950

changelog/0.17.1_2024-09-05/pull-5018

@@ -0,0 +1,13 @@
+Bugfix: Improve HTTP/2 support for REST backend
+
+If `rest-server` tried to gracefully shut down an HTTP/2 connection still in
+use by the client, it could result in the following error:
+
+```
+http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error
+```
+
+This issue has now been resolved.
+
+https://github.com/restic/restic/pull/5018
+https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367

changelog/0.17.2_2024-10-27/issue-4004

@@ -0,0 +1,12 @@
+Bugfix: Support container-level SAS/SAT tokens for Azure backend
+
+Restic previously expected SAS/SAT tokens to be generated at the account level,
+which prevented tokens created at the container level from being used to
+initialize a repository. This caused an error when attempting to initialize a
+repository with container-level tokens.
+
+Restic now supports both account-level and container-level SAS/SAT tokens for
+initializing a repository.
+
+https://github.com/restic/restic/issues/4004
+https://github.com/restic/restic/pull/5093

changelog/0.17.2_2024-10-27/issue-5050

@@ -0,0 +1,7 @@
+Bugfix: Return error if `tag` fails to lock repository
+
+Since restic 0.17.0, the `tag` command did not return an error when it failed
+to open or lock the repository. This issue has now been fixed.
+
+https://github.com/restic/restic/issues/5050
+https://github.com/restic/restic/pull/5056

changelog/0.17.2_2024-10-27/issue-5063

@@ -0,0 +1,12 @@
+Bugfix: Correctly `backup` extended metadata when using VSS on Windows
+
+On Windows, when creating a backup with the `--use-fs-snapshot` option, restic
+read extended metadata from the original filesystem path instead of from the
+snapshot. This could result in errors if files were removed during the backup
+process.
+
+This issue has now been resolved.
+
+https://github.com/restic/restic/issues/5063
+https://github.com/restic/restic/pull/5097
+https://github.com/restic/restic/pull/5099

changelog/0.17.2_2024-10-27/pull-5047

@@ -0,0 +1,8 @@
+Bugfix: Resolve potential error during concurrent cache cleanup
+
+When multiple restic processes ran concurrently, they could compete to remove
+obsolete snapshots from the local backend cache, sometimes leading to a "no
+such file or directory" error. Restic now suppresses this error to prevent
+issues during cache cleanup.
+
+https://github.com/restic/restic/pull/5047

changelog/0.17.2_2024-10-27/pull-5057

@@ -0,0 +1,24 @@
+Bugfix: Exclude irregular files from backups
+
+Since restic 0.17.1, files with the type `irregular` could mistakenly be included
+in snapshots, especially when backing up special file types on Windows that
+restic cannot process. This issue has now been fixed.
+
+Previously, this bug caused the `check` command to report errors like the
+following one:
+
+```
+  tree 12345678[...]: node "example.zip" with invalid type "irregular"
+```
+
+To repair affected snapshots, upgrade to restic 0.17.2 and run:
+
+```
+restic repair snapshots --forget
+```
+
+This will remove the `irregular` files from the snapshots (creating
+a new snapshot ID for each of the affected snapshots).
+
+https://github.com/restic/restic/pull/5057
+https://forum.restic.net/t/errors-found-by-check-1-invalid-type-irregular-2-ciphertext-verification-failed/8447/2

changelog/0.17.3_2024-11-08/issue-4971

@@ -0,0 +1,7 @@
+Bugfix: Fix unusable `mount` on macOS Sonoma
+
+On macOS Sonoma when using FUSE-T, it was not possible to access files in
+a mounted repository. This issue is now resolved.
+
+https://github.com/restic/restic/issues/4971
+https://github.com/restic/restic/pull/5048

changelog/0.17.3_2024-11-08/issue-5003

@@ -0,0 +1,14 @@
+Bugfix: Fix metadata errors during backup of removable disks on Windows
+
+Since restic 0.17.0, backing up removable disks on Windows could report
+errors with retrieving metadata like shown below.
+
+```
+error: incomplete metadata for d:\filename: get named security info failed with: Access is denied.
+```
+
+This has now been fixed.
+
+https://github.com/restic/restic/issues/5003
+https://github.com/restic/restic/pull/5123
+https://forum.restic.net/t/backing-up-a-folder-from-a-veracrypt-volume-brings-up-errors-since-restic-v17-0/8444

changelog/0.17.3_2024-11-08/issue-5107

@@ -0,0 +1,15 @@
+Bugfix: Fix metadata error on Windows for backups using VSS
+
+Since restic 0.17.2, when creating a backup on Windows using `--use-fs-snapshot`,
+restic would report an error like the following:
+
+```
+error: incomplete metadata for C:\: get EA failed while opening file handle for path \\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyXX\, with: The process cannot access the file because it is being used by another process.
+```
+
+This has now been fixed by correctly handling paths that refer to volume
+shadow copy snapshots.
+
+https://github.com/restic/restic/issues/5107
+https://github.com/restic/restic/pull/5110
+https://github.com/restic/restic/pull/5112

changelog/0.17.3_2024-11-08/pull-5096

@@ -0,0 +1,8 @@
+Enhancement: Allow `prune --dry-run` without lock
+
+The `prune --dry-run --no-lock` now allows performing a dry-run
+without locking the repository. Note that if the repository is
+modified concurrently, `prune` may return inaccurate statistics
+or errors.
+
+https://github.com/restic/restic/pull/5096

changelog/0.17.3_2024-11-08/pull-5101

@@ -0,0 +1,8 @@
+Bugfix: Do not retry load/list operation if SFTP connection is broken
+
+When using restic with the SFTP backend, backend operations that load a
+file or list files were retried even if the SFTP connection was broken.
+This has now been fixed.
+
+https://github.com/restic/restic/pull/5101
+https://forum.restic.net/t/restic-hanging-on-backup/8559

changelog/changelog-github.tmpl

@@ -15,7 +15,7 @@ Details
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }}
 {{ range $par := .Paragraphs }}
-   {{ $par }}
+{{ indent 3 $par }}
 {{ end }}
    {{ range $id := .Issues -}}
 {{ ` ` }}[#{{ $id }}](https://github.com/restic/restic/issues/{{ $id -}})

cmd/restic/cmd_backup.go

@@ -43,6 +43,7 @@ Exit status is 1 if there was a fatal error (no snapshot created).
 Exit status is 3 if some source data could not be read (incomplete snapshot created).
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	PreRun: func(_ *cobra.Command, _ []string) {
 		if backupOptions.Host == "" {
@@ -54,6 +55,7 @@ Exit status is 11 if the repository is already locked.
 			backupOptions.Host = hostname
 		}
 	},
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		term, cancel := setupTermstatus()
@@ -93,6 +95,7 @@ type BackupOptions struct {
 }
 
 var backupOptions BackupOptions
+var backupFSTestHook func(fs fs.FS) fs.FS
 
 // ErrInvalidSourceData is used to report an incomplete backup
 var ErrInvalidSourceData = errors.New("at least one source file could not be read")
@@ -596,6 +599,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		targets = []string{filename}
 	}
 
+	if backupFSTestHook != nil {
+		targetFS = backupFSTestHook(targetFS)
+	}
+
 	wg, wgCtx := errgroup.WithContext(ctx)
 	cancelCtx, cancel := context.WithCancel(wgCtx)
 	defer cancel()

cmd/restic/cmd_backup_integration_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -51,14 +52,14 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	opts := BackupOptions{UseFsSnapshot: useFsSnapshot}
 
 	// first backup
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	testListSnapshots(t, env.gopts, 1)
 
 	testRunCheck(t, env.gopts)
 	stat1 := dirStats(env.repo)
 
 	// second backup, implicit incremental
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	snapshotIDs := testListSnapshots(t, env.gopts, 2)
 
 	stat2 := dirStats(env.repo)
@@ -70,7 +71,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 	// third backup, explicit incremental
 	opts.Parent = snapshotIDs[0].String()
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	snapshotIDs = testListSnapshots(t, env.gopts, 3)
 
 	stat3 := dirStats(env.repo)
@@ -83,7 +84,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	for i, snapshotID := range snapshotIDs {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
-		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		testRunRestore(t, env.gopts, restoredir, snapshotID.String()+":"+toPathInSnapshot(filepath.Dir(env.testdata)))
 		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
 		rtest.Assert(t, diff == "", "directories are not equal: %v", diff)
 	}
@@ -91,6 +92,20 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 }
 
+func toPathInSnapshot(path string) string {
+	// use path as is on most platforms, but convert it on windows
+	if runtime.GOOS == "windows" {
+		// the path generated by the test is always local so take the shortcut
+		vol := filepath.VolumeName(path)
+		if vol[len(vol)-1] != ':' {
+			panic(fmt.Sprintf("unexpected path: %q", path))
+		}
+		path = vol[:len(vol)-1] + string(filepath.Separator) + path[len(vol)+1:]
+		path = filepath.ToSlash(path)
+	}
+	return path
+}
+
 func TestBackupWithRelativePath(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -111,6 +126,63 @@ func TestBackupWithRelativePath(t *testing.T) {
 	rtest.Assert(t, latestSn.Parent != nil && latestSn.Parent.Equal(firstSnapshotID), "second snapshot selected unexpected parent %v instead of %v", latestSn.Parent, firstSnapshotID)
 }
 
+type vssDeleteOriginalFS struct {
+	fs.FS
+	testdata   string
+	hasRemoved bool
+}
+
+func (f *vssDeleteOriginalFS) Lstat(name string) (os.FileInfo, error) {
+	if !f.hasRemoved {
+		// call Lstat to trigger snapshot creation
+		_, _ = f.FS.Lstat(name)
+		// nuke testdata
+		var err error
+		for i := 0; i < 3; i++ {
+			// The CI sometimes runs into "The process cannot access the file because it is being used by another process" errors
+			// thus try a few times to remove the data
+			err = os.RemoveAll(f.testdata)
+			if err == nil {
+				break
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+		if err != nil {
+			return nil, err
+		}
+		f.hasRemoved = true
+	}
+	return f.FS.Lstat(name)
+}
+
+func TestBackupVSS(t *testing.T) {
+	if runtime.GOOS != "windows" || fs.HasSufficientPrivilegesForVSS() != nil {
+		t.Skip("vss fs test can only be run on windows with admin privileges")
+	}
+
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{UseFsSnapshot: true}
+
+	var testFS *vssDeleteOriginalFS
+	backupFSTestHook = func(fs fs.FS) fs.FS {
+		testFS = &vssDeleteOriginalFS{
+			FS:       fs,
+			testdata: env.testdata,
+		}
+		return testFS
+	}
+	defer func() {
+		backupFSTestHook = nil
+	}()
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+	rtest.Equals(t, true, testFS.hasRemoved, "testdata was not removed")
+}
+
 func TestBackupParentSelection(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -499,7 +571,7 @@ func TestHardLink(t *testing.T) {
 	for i, snapshotID := range snapshotIDs {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
-		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		testRunRestore(t, env.gopts, restoredir, snapshotID.String())
 		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
 		rtest.Assert(t, diff == "", "directories are not equal %v", diff)
 

cmd/restic/cmd_cache.go

@@ -28,6 +28,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(_ *cobra.Command, args []string) error {
 		return runCache(cacheOptions, globalOptions, args)

cmd/restic/cmd_cat.go

@@ -12,6 +12,8 @@ import (
 	"github.com/restic/restic/internal/restic"
 )
 
+var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
+
 var cmdCat = &cobra.Command{
 	Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
 	Short: "Print internal objects to stdout",
@@ -25,11 +27,14 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCat(cmd.Context(), globalOptions, args)
 	},
+	ValidArgs: catAllowedCmds,
 }
 
 func init() {
@@ -37,21 +42,19 @@ func init() {
 }
 
 func validateCatArgs(args []string) error {
-	var allowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
-
 	if len(args) < 1 {
 		return errors.Fatal("type not specified")
 	}
 
 	validType := false
-	for _, v := range allowedCmds {
+	for _, v := range catAllowedCmds {
 		if v == args[0] {
 			validType = true
 			break
 		}
 	}
 	if !validType {
-		return errors.Fatalf("invalid type %q, must be one of [%s]", args[0], strings.Join(allowedCmds, "|"))
+		return errors.Fatalf("invalid type %q, must be one of [%s]", args[0], strings.Join(catAllowedCmds, "|"))
 	}
 
 	if args[0] != "masterkey" && args[0] != "config" && len(args) != 2 {

cmd/restic/cmd_check.go

@@ -39,7 +39,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		term, cancel := setupTermstatus()

cmd/restic/cmd_copy.go

@@ -38,7 +38,10 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCopy(cmd.Context(), copyOptions, globalOptions, args)
 	},

cmd/restic/cmd_copy_integration_test.go

@@ -62,11 +62,11 @@ func TestCopy(t *testing.T) {
 	for i, snapshotID := range snapshotIDs {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		origRestores[restoredir] = struct{}{}
-		testRunRestore(t, env.gopts, restoredir, snapshotID)
+		testRunRestore(t, env.gopts, restoredir, snapshotID.String())
 	}
 	for i, snapshotID := range copiedSnapshotIDs {
 		restoredir := filepath.Join(env2.base, fmt.Sprintf("restore%d", i))
-		testRunRestore(t, env2.gopts, restoredir, snapshotID)
+		testRunRestore(t, env2.gopts, restoredir, snapshotID.String())
 		foundMatch := false
 		for cmpdir := range origRestores {
 			diff := directoriesContentsDiff(restoredir, cmpdir)

cmd/restic/cmd_debug.go

@@ -29,8 +29,10 @@ import (
 )
 
 var cmdDebug = &cobra.Command{
-	Use:   "debug",
-	Short: "Debug commands",
+	Use:               "debug",
+	Short:             "Debug commands",
+	GroupID:           cmdGroupDefault,
+	DisableAutoGenTag: true,
 }
 
 var cmdDebugDump = &cobra.Command{
@@ -47,6 +49,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/restic/cmd_diff.go

@@ -43,7 +43,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runDiff(cmd.Context(), diffOptions, globalOptions, args)
@@ -177,6 +179,10 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 	}
 
 	for _, node := range tree.Nodes {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
 		name := path.Join(prefix, node.Name)
 		if node.Type == "dir" {
 			name += "/"
@@ -187,13 +193,13 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 
 		if node.Type == "dir" {
 			err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
-			if err != nil {
+			if err != nil && err != context.Canceled {
 				Warnf("error: %v\n", err)
 			}
 		}
 	}
 
-	return nil
+	return ctx.Err()
 }
 
 func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
@@ -204,17 +210,21 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest
 	}
 
 	for _, node := range tree.Nodes {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
 		addBlobs(blobs, node)
 
 		if node.Type == "dir" {
 			err := c.collectDir(ctx, blobs, *node.Subtree)
-			if err != nil {
+			if err != nil && err != context.Canceled {
 				Warnf("error: %v\n", err)
 			}
 		}
 	}
 
-	return nil
+	return ctx.Err()
 }
 
 func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) {
@@ -255,6 +265,10 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 	tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2)
 
 	for _, name := range names {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
 		node1, t1 := tree1Nodes[name]
 		node2, t2 := tree2Nodes[name]
 
@@ -304,7 +318,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 				} else {
 					err = c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree)
 				}
-				if err != nil {
+				if err != nil && err != context.Canceled {
 					Warnf("error: %v\n", err)
 				}
 			}
@@ -318,7 +332,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 
 			if node1.Type == "dir" {
 				err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
-				if err != nil {
+				if err != nil && err != context.Canceled {
 					Warnf("error: %v\n", err)
 				}
 			}
@@ -332,14 +346,14 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 
 			if node2.Type == "dir" {
 				err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
-				if err != nil {
+				if err != nil && err != context.Canceled {
 					Warnf("error: %v\n", err)
 				}
 			}
 		}
 	}
 
-	return nil
+	return ctx.Err()
 }
 
 func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {

cmd/restic/cmd_dump.go

@@ -38,7 +38,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runDump(cmd.Context(), dumpOptions, globalOptions, args)
@@ -85,6 +87,10 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade
 	item := filepath.Join(prefix, pathComponents[0])
 	l := len(pathComponents)
 	for _, node := range tree.Nodes {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
 		// If dumping something in the highest level it will just take the
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {

cmd/restic/cmd_features.go

@@ -31,7 +31,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	Hidden:            true,
+	GroupID:           cmdGroupAdvanced,
 	DisableAutoGenTag: true,
 	RunE: func(_ *cobra.Command, args []string) error {
 		if len(args) != 0 {

cmd/restic/cmd_find.go

@@ -37,7 +37,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runFind(cmd.Context(), findOptions, globalOptions, args)
@@ -377,6 +379,10 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 
 		if node.Type == "file" && f.blobIDs != nil {
 			for _, id := range node.Content {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+
 				idStr := id.String()
 				if _, ok := f.blobIDs[idStr]; !ok {
 					// Look for short ID form

cmd/restic/cmd_forget.go

@@ -39,7 +39,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		term, cancel := setupTermstatus()
@@ -246,6 +248,10 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 		printer.P("Applying Policy: %v\n", policy)
 
 		for k, snapshotGroup := range snapshotGroups {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+
 			if gopts.Verbose >= 1 && !gopts.JSON {
 				err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
 				if err != nil {

cmd/restic/cmd_init.go

@@ -26,6 +26,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runInit(cmd.Context(), initOptions, globalOptions, args)

cmd/restic/cmd_init_integration_test.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/restic/restic/internal/repository"
@@ -16,6 +18,11 @@ func testRunInit(t testing.TB, opts GlobalOptions) {
 
 	rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil))
 	t.Logf("repository initialized at %v", opts.Repo)
+
+	// create temporary junk files to verify that restic does not trip over them
+	for _, path := range []string{"index", "snapshots", "keys", "locks", filepath.Join("data", "00")} {
+		rtest.OK(t, os.WriteFile(filepath.Join(opts.Repo, path, "tmp12345"), []byte("junk file"), 0o600))
+	}
 }
 
 func TestInitCopyChunkerParams(t *testing.T) {

cmd/restic/cmd_key.go

@@ -11,6 +11,8 @@ var cmdKey = &cobra.Command{
 The "key" command allows you to set multiple access keys or passwords
 per repository.
 	`,
+	DisableAutoGenTag: true,
+	GroupID:           cmdGroupDefault,
 }
 
 func init() {

cmd/restic/cmd_key_add.go

@@ -23,6 +23,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 	`,
 	DisableAutoGenTag: true,
 }

cmd/restic/cmd_key_list.go

@@ -27,6 +27,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 	`,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/restic/cmd_key_passwd.go

@@ -23,6 +23,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 	`,
 	DisableAutoGenTag: true,
 }

cmd/restic/cmd_key_remove.go

@@ -24,6 +24,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 	`,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/restic/cmd_list.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"strings"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository/index"
@@ -10,8 +11,11 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var listAllowedArgs = []string{"blobs", "packs", "index", "snapshots", "keys", "locks"}
+var listAllowedArgsUseString = strings.Join(listAllowedArgs, "|")
+
 var cmdList = &cobra.Command{
-	Use:   "list [flags] [blobs|packs|index|snapshots|keys|locks]",
+	Use:   "list [flags] [" + listAllowedArgsUseString + "]",
 	Short: "List objects in the repository",
 	Long: `
 The "list" command allows listing objects in the repository based on type.
@@ -23,11 +27,15 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
+	GroupID:           cmdGroupDefault,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runList(cmd.Context(), globalOptions, args)
 	},
+	ValidArgs: listAllowedArgs,
+	Args:      cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
 }
 
 func init() {

cmd/restic/cmd_ls.go

@@ -43,8 +43,10 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
+	GroupID:           cmdGroupDefault,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runLs(cmd.Context(), lsOptions, globalOptions, args)
 	},

cmd/restic/cmd_migrate.go

@@ -26,8 +26,10 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
+	GroupID:           cmdGroupDefault,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		term, cancel := setupTermstatus()
 		defer cancel()
@@ -74,8 +76,10 @@ func checkMigrations(ctx context.Context, repo restic.Repository, printer progre
 func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string, term *termstatus.Terminal, printer progress.Printer) error {
 	var firsterr error
 	for _, name := range args {
+		found := false
 		for _, m := range migrations.All {
 			if m.Name() == name {
+				found = true
 				ok, reason, err := m.Check(ctx, repo)
 				if err != nil {
 					return err
@@ -119,6 +123,9 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio
 				printer.P("migration %v: success\n", m.Name())
 			}
 		}
+		if !found {
+			printer.E("unknown migration %v", name)
+		}
 	}
 
 	return firsterr

cmd/restic/cmd_mount.go

@@ -68,8 +68,10 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
+	GroupID:           cmdGroupDefault,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runMount(cmd.Context(), mountOptions, globalOptions, args)
 	},

cmd/restic/cmd_mount_integration_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	systemFuse "github.com/anacrolix/fuse"
+	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -205,6 +206,11 @@ func TestMountSameTimestamps(t *testing.T) {
 		t.Skip("Skipping fuse tests")
 	}
 
+	debugEnabled := debug.TestLogToStderr(t)
+	if debugEnabled {
+		defer debug.TestDisableLog(t)
+	}
+
 	env, cleanup := withTestEnvironment(t)
 	// must list snapshots more than once
 	env.gopts.backendTestHook = nil

cmd/restic/cmd_options.go

@@ -20,7 +20,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	Hidden:            true,
+	GroupID:           cmdGroupAdvanced,
 	DisableAutoGenTag: true,
 	Run: func(_ *cobra.Command, _ []string) {
 		fmt.Printf("All Extended Options:\n")

cmd/restic/cmd_prune.go

@@ -32,7 +32,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, _ []string) error {
 		term, cancel := setupTermstatus()
@@ -147,7 +149,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term
 		return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
 	}
 
-	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	if gopts.NoLock && !opts.DryRun {
+		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for prune command")
+	}
+
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_prune_integration_test.go

@@ -146,10 +146,9 @@ func TestPruneWithDamagedRepository(t *testing.T) {
 		env.gopts.backendTestHook = oldHook
 	}()
 	// prune should fail
-	rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+	rtest.Equals(t, repository.ErrPacksMissing, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
 		return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term)
-	}) == repository.ErrPacksMissing,
-		"prune should have reported index not complete error")
+	}), "prune should have reported index not complete error")
 }
 
 // Test repos for edge cases

cmd/restic/cmd_recover.go

@@ -26,7 +26,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, _ []string) error {
 		return runRecover(cmd.Context(), globalOptions)
@@ -118,6 +120,10 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return nil
 	}
 
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
 	tree := restic.NewTree(len(roots))
 	for id := range roots {
 		var subtreeID = id

cmd/restic/cmd_repair.go

@@ -5,8 +5,10 @@ import (
 )
 
 var cmdRepair = &cobra.Command{
-	Use:   "repair",
-	Short: "Repair the repository",
+	Use:               "repair",
+	Short:             "Repair the repository",
+	GroupID:           cmdGroupDefault,
+	DisableAutoGenTag: true,
 }
 
 func init() {

cmd/restic/cmd_repair_index.go

@@ -23,6 +23,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, _ []string) error {

cmd/restic/cmd_repair_packs.go

@@ -27,6 +27,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/restic/cmd_repair_snapshots.go

@@ -41,6 +41,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -91,6 +92,10 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 	// - files whose contents are not fully available  (-> file will be modified)
 	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
 		RewriteNode: func(node *restic.Node, path string) *restic.Node {
+			if node.Type == "irregular" || node.Type == "" {
+				Verbosef("  file %q: removed node with invalid type %q\n", path, node.Type)
+				return nil
+			}
 			if node.Type != "file" {
 				return node
 			}

cmd/restic/cmd_restore.go

@@ -36,7 +36,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		term, cancel := setupTermstatus()
@@ -164,9 +166,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 
 	totalErrors := 0
 	res.Error = func(location string, err error) error {
-		msg.E("ignoring error for %s: %s\n", location, err)
 		totalErrors++
-		return nil
+		return progress.Error(location, err)
 	}
 	res.Warn = func(message string) {
 		msg.E("Warning: %s\n", message)
@@ -221,7 +222,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target)
 	}
 
-	err = res.RestoreTo(ctx, opts.Target)
+	countRestoredFiles, err := res.RestoreTo(ctx, opts.Target)
 	if err != nil {
 		return err
 	}
@@ -238,7 +239,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		}
 		var count int
 		t0 := time.Now()
-		count, err = res.VerifyFiles(ctx, opts.Target)
+		bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON && stdoutIsTerminal(), 0, "files verified", term)
+		count, err = res.VerifyFiles(ctx, opts.Target, countRestoredFiles, bar)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_restore_integration_test.go

@@ -18,17 +18,17 @@ import (
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) {
+func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID string) {
 	testRunRestoreExcludes(t, opts, dir, snapshotID, nil)
 }
 
-func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
+func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID string, excludes []string) {
 	opts := RestoreOptions{
 		Target: dir,
 	}
 	opts.Excludes = excludes
 
-	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
+	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID, opts, gopts))
 }
 
 func testRunRestoreAssumeFailure(snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
@@ -198,7 +198,7 @@ func TestRestoreFilter(t *testing.T) {
 	snapshotID := testListSnapshots(t, env.gopts, 1)[0]
 
 	// no restore filter should restore all files
-	testRunRestore(t, env.gopts, filepath.Join(env.base, "restore0"), snapshotID)
+	testRunRestore(t, env.gopts, filepath.Join(env.base, "restore0"), snapshotID.String())
 	for _, testFile := range testfiles {
 		rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size)))
 	}
@@ -220,7 +220,7 @@ func TestRestoreFilter(t *testing.T) {
 
 	// restore with excludes
 	restoredir := filepath.Join(env.base, "restore-with-excludes")
-	testRunRestoreExcludes(t, env.gopts, restoredir, snapshotID, excludePatterns)
+	testRunRestoreExcludes(t, env.gopts, restoredir, snapshotID.String(), excludePatterns)
 	testRestoredFileExclusions(t, restoredir)
 
 	// Create an exclude file with some patterns
@@ -340,7 +340,7 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
 
 	_ = withRestoreGlobalOptions(func() error {
 		globalOptions.stderr = io.Discard
-		testRunRestore(t, env.gopts, filepath.Join(env.base, "restore"), snapshots[0])
+		testRunRestore(t, env.gopts, filepath.Join(env.base, "restore"), snapshots[0].String())
 		return nil
 	})
 

cmd/restic/cmd_rewrite.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -42,7 +41,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runRewrite(cmd.Context(), rewriteOptions, globalOptions, args)
@@ -138,7 +139,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 			if selectByName(path) {
 				return node
 			}
-			Verbosef(fmt.Sprintf("excluding %s\n", path))
+			Verbosef("excluding %s\n", path)
 			return nil
 		}
 

cmd/restic/cmd_self_update.go

@@ -28,6 +28,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/restic/cmd_snapshots.go

@@ -27,7 +27,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args)
@@ -81,6 +83,10 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
 	}
 
 	for k, list := range snapshotGroups {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
 		if opts.Last {
 			// This branch should be removed in the same time
 			// that --last.
@@ -101,6 +107,10 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
 	}
 
 	for k, list := range snapshotGroups {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
 		if grouped {
 			err := PrintSnapshotGroupHeader(globalOptions.stdout, k)
 			if err != nil {

cmd/restic/cmd_stats.go

@@ -53,7 +53,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runStats(cmd.Context(), statsOptions, globalOptions, args)
@@ -70,10 +72,20 @@ type StatsOptions struct {
 
 var statsOptions StatsOptions
 
+func must(err error) {
+	if err != nil {
+		panic(fmt.Sprintf("error during setup: %v", err))
+	}
+}
+
 func init() {
 	cmdRoot.AddCommand(cmdStats)
 	f := cmdStats.Flags()
 	f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
+	must(cmdStats.RegisterFlagCompletionFunc("mode", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) {
+		return []string{countModeRestoreSize, countModeUniqueFilesByContents, countModeBlobsPerFile, countModeRawData}, cobra.ShellCompDirectiveDefault
+	}))
+
 	initMultiSnapshotFilter(f, &statsOptions.SnapshotFilter, true)
 }
 

cmd/restic/cmd_tag.go

@@ -29,7 +29,9 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runTag(cmd.Context(), tagOptions, globalOptions, args)
@@ -108,7 +110,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
 	Verbosef("create exclusive lock for repository\n")
 	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
-		return nil
+		return err
 	}
 	defer unlock()
 

cmd/restic/cmd_unlock.go

@@ -19,6 +19,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
+	GroupID:           cmdGroupDefault,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, _ []string) error {
 		return runUnlock(cmd.Context(), unlockOptions, globalOptions)

cmd/restic/global.go

@@ -47,7 +47,7 @@ import (
 // to a missing backend storage location or config file
 var ErrNoRepository = errors.New("repository does not exist")
 
-var version = "0.17.0"
+var version = "0.17.3"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -140,6 +140,7 @@ func init() {
 	f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
 	f.StringVar(&globalOptions.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests")
+	f.DurationVar(&globalOptions.StuckRequestTimeout, "stuck-request-timeout", 5*time.Minute, "`duration` after which to retry stuck requests")
 	// Use our "generate" command instead of the cobra provided "completion" command
 	cmdRoot.CompletionOptions.DisableDefaultCmd = true
 
@@ -493,7 +494,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
 		}
 	}
 	if err != nil {
-		if errors.IsFatal(err) {
+		if errors.IsFatal(err) || errors.Is(err, repository.ErrNoKeyFound) {
 			return nil, err
 		}
 		return nil, errors.Fatalf("%s", err)

cmd/restic/integration_helpers_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync"
 	"testing"
 
@@ -168,6 +169,16 @@ type testEnvironment struct {
 	gopts                                   GlobalOptions
 }
 
+type logOutputter struct {
+	t testing.TB
+}
+
+func (l *logOutputter) Write(p []byte) (n int, err error) {
+	l.t.Helper()
+	l.t.Log(strings.TrimSuffix(string(p), "\n"))
+	return len(p), nil
+}
+
 // withTestEnvironment creates a test environment and returns a cleanup
 // function which removes it.
 func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
@@ -200,8 +211,11 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 		Quiet:    true,
 		CacheDir: env.cache,
 		password: rtest.TestPassword,
-		stdout:   os.Stdout,
-		stderr:   os.Stderr,
+		// stdout and stderr are written to by Warnf etc. That is the written data
+		// usually consists of one or multiple lines and therefore can be handled well
+		// by t.Log.
+		stdout:   &logOutputter{t},
+		stderr:   &logOutputter{t},
 		extended: make(options.Options),
 
 		// replace this hook with "nil" if listing a filetype more than once is necessary

cmd/restic/integration_test.go

@@ -35,7 +35,7 @@ func TestCheckRestoreNoLock(t *testing.T) {
 	testRunCheck(t, env.gopts)
 
 	snapshotIDs := testListSnapshots(t, env.gopts, 4)
-	testRunRestore(t, env.gopts, filepath.Join(env.base, "restore"), snapshotIDs[0])
+	testRunRestore(t, env.gopts, filepath.Join(env.base, "restore"), snapshotIDs[0].String())
 }
 
 // a listOnceBackend only allows listing once per filetype
@@ -80,7 +80,7 @@ func TestListOnce(t *testing.T) {
 	defer cleanup()
 
 	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
-		return newListOnceBackend(r), nil
+		return newOrderedListOnceBackend(r), nil
 	}
 	pruneOpts := PruneOptions{MaxUnused: "0"}
 	checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
@@ -148,7 +148,7 @@ func TestFindListOnce(t *testing.T) {
 	defer cleanup()
 
 	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
-		return newListOnceBackend(r), nil
+		return newOrderedListOnceBackend(r), nil
 	}
 
 	testSetupBackupData(t, env)

cmd/restic/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/options"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 )
 
@@ -82,6 +83,22 @@ The full documentation can be found at https://restic.readthedocs.io/ .
 	},
 }
 
+var cmdGroupDefault = "default"
+var cmdGroupAdvanced = "advanced"
+
+func init() {
+	cmdRoot.AddGroup(
+		&cobra.Group{
+			ID:    cmdGroupDefault,
+			Title: "Available Commands:",
+		},
+		&cobra.Group{
+			ID:    cmdGroupAdvanced,
+			Title: "Advanced Options:",
+		},
+	)
+}
+
 // Distinguish commands that need the password from those that work without,
 // so we don't run $RESTIC_PASSWORD_COMMAND for no reason (it might prompt the
 // user for authentication).
@@ -138,6 +155,8 @@ func main() {
 		fmt.Fprintf(os.Stderr, "Warning: %v\n", err)
 	case errors.IsFatal(err):
 		fmt.Fprintf(os.Stderr, "%v\n", err)
+	case errors.Is(err, repository.ErrNoKeyFound):
+		fmt.Fprintf(os.Stderr, "Fatal: %v\n", err)
 	case err != nil:
 		fmt.Fprintf(os.Stderr, "%+v\n", err)
 
@@ -160,6 +179,8 @@ func main() {
 		exitCode = 10
 	case restic.IsAlreadyLocked(err):
 		exitCode = 11
+	case errors.Is(err, repository.ErrNoKeyFound):
+		exitCode = 12
 	case errors.Is(err, context.Canceled):
 		exitCode = 130
 	default:

cmd/restic/secondary_repo.go

@@ -50,7 +50,7 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo
 	f.StringVarP(&opts.PasswordFile, "from-password-file", "", "", "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)")
 	f.StringVarP(&opts.KeyHint, "from-key-hint", "", "", "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)")
 	f.StringVarP(&opts.PasswordCommand, "from-password-command", "", "", "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)")
-	f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository, must be passed to every restic command (insecure)")
+	f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository (insecure)")
 
 	opts.Repo = os.Getenv("RESTIC_FROM_REPOSITORY")
 	opts.RepositoryFile = os.Getenv("RESTIC_FROM_REPOSITORY_FILE")

doc/030_preparing_a_new_repo.rst

@@ -249,28 +249,22 @@ while creating the bucket.
     $ export AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
     $ export AWS_SECRET_ACCESS_KEY=<MY_SECRET_ACCESS_KEY>
 
+When using temporary credentials make sure to include the session token via
+the environment variable ``AWS_SESSION_TOKEN``.
+
 You can then easily initialize a repository that uses your Amazon S3 as
-a backend. If the bucket does not exist it will be created in the
-default location:
+a backend. Make sure to use the endpoint for the correct region. The example
+uses ``us-east-1``. If the bucket does not exist it will be created in that region:
 
 .. code-block:: console
 
-    $ restic -r s3:s3.amazonaws.com/bucket_name init
+    $ restic -r s3:s3.us-east-1.amazonaws.com/bucket_name init
     enter password for new repository:
     enter password again:
-    created restic repository eefee03bbd at s3:s3.amazonaws.com/bucket_name
+    created restic repository eefee03bbd at s3:s3.us-east-1.amazonaws.com/bucket_name
     Please note that knowledge of your password is required to access the repository.
     Losing your password means that your data is irrecoverably lost.
 
-If needed, you can manually specify the region to use by either setting the
-environment variable ``AWS_DEFAULT_REGION`` or calling restic with an option
-parameter like ``-o s3.region="us-east-1"``. If the region is not specified,
-the default region is used. Afterwards, the S3 server (at least for AWS,
-``s3.amazonaws.com``) will redirect restic to the correct endpoint.
-
-When using temporary credentials make sure to include the session token via
-then environment variable ``AWS_SESSION_TOKEN``.
-
 Until version 0.8.0, restic used a default prefix of ``restic``, so the files
 in the bucket were placed in a directory named ``restic``. If you want to
 access a repository created with an older version of restic, specify the path
@@ -278,25 +272,14 @@ after the bucket name like this:
 
 .. code-block:: console
 
-    $ restic -r s3:s3.amazonaws.com/bucket_name/restic [...]
+    $ restic -r s3:s3.us-east-1.amazonaws.com/bucket_name/restic [...]
 
-For an S3-compatible server that is not Amazon (like Minio, see below),
-or is only available via HTTP, you can specify the URL to the server
-like this: ``s3:http://server:port/bucket_name``.
-          
 .. note:: restic expects `path-style URLs <https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html>`__
-          like for example ``s3.us-west-2.amazonaws.com/bucket_name``.
+          like for example ``s3.us-west-2.amazonaws.com/bucket_name`` for Amazon S3.
           Virtual-hosted–style URLs like ``bucket_name.s3.us-west-2.amazonaws.com``,
           where the bucket name is part of the hostname are not supported. These must
           be converted to path-style URLs instead, for example ``s3.us-west-2.amazonaws.com/bucket_name``.
-
-.. note:: Certain S3-compatible servers do not properly implement the
-          ``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these
-          backends, as a temporary workaround, you can provide the
-          ``-o s3.list-objects-v1=true`` option to use the older
-          ``ListObjects`` API instead. This option may be removed in future
-          versions of restic.
-
+          See below for configuration options for S3-compatible storage from other providers.
 
 Minio Server
 ************
@@ -321,81 +304,66 @@ this command.
 
 .. code-block:: console
 
-    $ ./restic -r s3:http://localhost:9000/restic init
+    $ restic -r s3:http://localhost:9000/restic init
     enter password for new repository:
     enter password again:
-    created restic repository 6ad29560f5 at s3:http://localhost:9000/restic1
+    created restic repository 6ad29560f5 at s3:http://localhost:9000/restic
     Please note that knowledge of your password is required to access
     the repository. Losing your password means that your data is irrecoverably lost.
 
+S3-compatible Storage
+*********************
+
+For an S3-compatible server that is not Amazon, you can specify the URL to the server
+like this: ``s3:https://server:port/bucket_name``.
+
+If needed, you can manually specify the region to use by either setting the
+environment variable ``AWS_DEFAULT_REGION`` or calling restic with an option
+parameter like ``-o s3.region="us-east-1"``. If the region is not specified,
+the default region ``us-east-1`` is used.
+
+To select between path-style and virtual-hosted access, the extended option
+``-o s3.bucket-lookup=auto`` can be used. It supports the following values:
+
+- ``auto``: Default behavior. Uses ``dns`` for Amazon and Google endpoints. Uses
+  ``path`` for all other endpoints
+- ``dns``: Use virtual-hosted-style bucket access
+- ``path``: Use path-style bucket access
+
+Certain S3-compatible servers do not properly implement the ``ListObjectsV2`` API,
+most notably Ceph versions before v14.2.5. On these backends, as a temporary
+workaround, you can provide the ``-o s3.list-objects-v1=true`` option to use the
+older ``ListObjects`` API instead. This option may be removed in future versions
+of restic.
+
 Wasabi
-************
+******
 
-`Wasabi <https://wasabi.com>`__ is a low cost Amazon S3 conformant object storage provider.
-Due to its S3 conformance, Wasabi can be used as a storage provider for a restic repository.
+S3 storage from `Wasabi <https://wasabi.com>`__ can be used as follows.
 
--  Create a Wasabi bucket using the `Wasabi Console <https://console.wasabisys.com>`__.
--  Determine the correct Wasabi service URL for your bucket `here <https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031-What-are-the-service-URLs-for-Wasabi-s-different-regions->`__.
-
-You must first setup the following environment variables with the
-credentials of your Wasabi account.
+- Determine the correct Wasabi service URL for your bucket `here <https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031-What-are-the-service-URLs-for-Wasabi-s-different-regions->`__.
+- Set environment variables with the necessary account credentials
 
 .. code-block:: console
 
     $ export AWS_ACCESS_KEY_ID=<YOUR-WASABI-ACCESS-KEY-ID>
     $ export AWS_SECRET_ACCESS_KEY=<YOUR-WASABI-SECRET-ACCESS-KEY>
-
-Now you can easily initialize restic to use Wasabi as a backend with
-this command.
-
-.. code-block:: console
-
-    $ ./restic -r s3:https://<WASABI-SERVICE-URL>/<WASABI-BUCKET-NAME> init
-    enter password for new repository:
-    enter password again:
-    created restic repository xxxxxxxxxx at s3:https://<WASABI-SERVICE-URL>/<WASABI-BUCKET-NAME>
-    Please note that knowledge of your password is required to access
-    the repository. Losing your password means that your data is irrecoverably lost.
+    $ restic -r s3:https://<WASABI-SERVICE-URL>/<WASABI-BUCKET-NAME> init
 
 Alibaba Cloud (Aliyun) Object Storage System (OSS)
 **************************************************
 
-`Alibaba OSS <https://www.alibabacloud.com/product/object-storage-service>`__ is an
-encrypted, secure, cost-effective, and easy-to-use object storage
-service that enables you to store, back up, and archive large amounts
-of data in the cloud.
+S3 storage from `Alibaba OSS <https://www.alibabacloud.com/product/object-storage-service>`__ can be used as follows.
 
-Alibaba OSS is S3 compatible so it can be used as a storage provider
-for a restic repository with a couple of extra parameters.
-
--  Determine the correct `Alibaba OSS region endpoint <https://www.alibabacloud.com/help/en/object-storage-service/latest/regions-and-endpoints>`__ - this will be something like ``oss-eu-west-1.aliyuncs.com``
--  You'll need the region name too - this will be something like ``oss-eu-west-1``
-
-You must first setup the following environment variables with the
-credentials of your Alibaba OSS account.
+- Determine the correct `Alibaba OSS region endpoint <https://www.alibabacloud.com/help/en/object-storage-service/latest/regions-and-endpoints>`__ - this will be something like ``oss-eu-west-1.aliyuncs.com``
+- You will need the region name too - this will be something like ``oss-eu-west-1``
+- Set environment variables with the necessary account credentials
 
 .. code-block:: console
 
     $ export AWS_ACCESS_KEY_ID=<YOUR-OSS-ACCESS-KEY-ID>
     $ export AWS_SECRET_ACCESS_KEY=<YOUR-OSS-SECRET-ACCESS-KEY>
-
-Now you can easily initialize restic to use Alibaba OSS as a backend with
-this command.
-
-.. code-block:: console
-
-    $ ./restic -o s3.bucket-lookup=dns -o s3.region=<OSS-REGION> -r s3:https://<OSS-ENDPOINT>/<OSS-BUCKET-NAME> init
-    enter password for new backend:
-    enter password again:
-    created restic backend xxxxxxxxxx at s3:https://<OSS-ENDPOINT>/<OSS-BUCKET-NAME>
-    Please note that knowledge of your password is required to access
-    the repository. Losing your password means that your data is irrecoverably lost.
-
-For example with an actual endpoint:
-
-.. code-block:: console
-
-    $ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
+    $ restic -o s3.bucket-lookup=dns -o s3.region=<OSS-REGION> -r s3:https://<OSS-ENDPOINT>/<OSS-BUCKET-NAME> init
 
 OpenStack Swift
 ***************
@@ -487,9 +455,11 @@ Backblaze B2
    than using the Backblaze B2 backend directly.
 
    Different from the B2 backend, restic's S3 backend will only hide no longer
-   necessary files. Thus, make sure to setup lifecycle rules to eventually
-   delete hidden files. The lifecycle setting "Keep only the last version of the file" 
-   will keep only the most current version of a file. Read the [Backblaze documentation](https://www.backblaze.com/docs/cloud-storage-lifecycle-rules).
+   necessary files. By default, Backblaze B2 retains all of the different versions of the
+   files and "hides" the older versions. Thus, to free space occupied by hidden files,
+   it is **recommended** to use the B2 lifecycle "Keep only the last version of the file".
+   The previous version of the file is "hidden" for one day and then deleted automatically
+   by B2. More details at the [Backblaze documentation](https://www.backblaze.com/docs/cloud-storage-lifecycle-rules).
 
 Restic can backup data to any Backblaze B2 bucket. You need to first setup the
 following environment variables with the credentials you can find in the

doc/040_backup.rst

@@ -584,11 +584,13 @@ Reading data from a command
 Sometimes, it can be useful to directly save the output of a program, for example,
 ``mysqldump`` so that the SQL can later be restored. Restic supports this mode
 of operation; just supply the option ``--stdin-from-command`` when using the
-``backup`` action, and write the command in place of the files/directories:
+``backup`` action, and write the command in place of the files/directories. To prevent
+restic from interpreting the arguments for the command, make sure to add ``--`` before
+the command starts:
 
 .. code-block:: console
 
-    $ restic -r /srv/restic-repo backup --stdin-from-command mysqldump [...]
+    $ restic -r /srv/restic-repo backup --stdin-from-command -- mysqldump --host example mydb [...]
 
 This command creates a new snapshot based on the standard output of ``mysqldump``.
 By default, the command's standard output is saved in a file named ``stdin``.
@@ -596,7 +598,7 @@ A different name can be specified with ``--stdin-filename``:
 
 .. code-block:: console
 
-    $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command mysqldump [...]
+    $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command -- mysqldump --host example mydb [...]
 
 Restic uses the command exit code to determine whether the command succeeded. A
 non-zero exit code from the command causes restic to cancel the backup. This causes
@@ -684,6 +686,30 @@ created as it would only be written at the very (successful) end of
 the backup operation.  Previous snapshots will still be there and will still
 work.
 
+Exit status codes
+*****************
+
+Restic returns an exit status code after the backup command is run:
+
+* 0 when the backup was successful (snapshot with all source files created)
+* 1 when there was a fatal error (no snapshot created)
+* 3 when some source files could not be read (incomplete snapshot with remaining files created)
+* further exit codes are documented in :ref:`exit-codes`.
+
+Fatal errors occur for example when restic is unable to write to the backup destination, when
+there are network connectivity issues preventing successful communication, or when an invalid
+password or command line argument is provided. When restic returns this exit status code, one
+should not expect a snapshot to have been created.
+
+Source file read errors occur when restic fails to read one or more files or directories that
+it was asked to back up, e.g. due to permission problems. Restic displays the number of source
+file read errors that occurred while running the backup. If there are errors of this type,
+restic will still try to complete the backup run with all the other files, and create a
+snapshot that then contains all but the unreadable files.
+
+For use of these exit status codes in scripts and other automation tools, see :ref:`exit-codes`.
+To manually inspect the exit code in e.g. Linux, run ``echo $?``.
+
 Environment Variables
 *********************
 
@@ -702,6 +728,7 @@ environment variables. The following lists these environment variables:
     RESTIC_TLS_CLIENT_CERT              Location of TLS client certificate and private key (replaces --tls-client-cert)
     RESTIC_CACHE_DIR                    Location of the cache directory
     RESTIC_COMPRESSION                  Compression mode (only available for repository format version 2)
+    RESTIC_HOST                         Only consider snapshots for this host / Set the hostname for the snapshot manually (replaces --host)
     RESTIC_PROGRESS_FPS                 Frames per second by which the progress bar is updated
     RESTIC_PACK_SIZE                    Target size for pack files
     RESTIC_READ_CONCURRENCY             Concurrency for file reads
@@ -771,26 +798,3 @@ See :ref:`caching` for the rules concerning cache locations when
 The external programs that restic may execute include ``rclone`` (for rclone
 backends) and ``ssh`` (for the SFTP backend). These may respond to further
 environment variables and configuration files; see their respective manuals.
-
-Exit status codes
-*****************
-
-Restic returns one of the following exit status codes after the backup command is run:
-
-* 0 when the backup was successful (snapshot with all source files created)
-* 1 when there was a fatal error (no snapshot created)
-* 3 when some source files could not be read (incomplete snapshot with remaining files created)
-
-Fatal errors occur for example when restic is unable to write to the backup destination, when
-there are network connectivity issues preventing successful communication, or when an invalid
-password or command line argument is provided. When restic returns this exit status code, one
-should not expect a snapshot to have been created.
-
-Source file read errors occur when restic fails to read one or more files or directories that
-it was asked to back up, e.g. due to permission problems. Restic displays the number of source
-file read errors that occurred while running the backup. If there are errors of this type,
-restic will still try to complete the backup run with all the other files, and create a
-snapshot that then contains all but the unreadable files.
-
-One can use these exit status codes in scripts and other automation tools, to make them aware of
-the outcome of the backup run. To manually inspect the exit code in e.g. Linux, run ``echo $?``.

doc/045_working_with_repos.rst

@@ -305,6 +305,13 @@ In order to preview the changes which ``rewrite`` would make, you can use the
 modifying the repository. Instead restic will only print the actions it would
 perform.
 
+.. note:: The ``rewrite`` command verifies that it does not modify snapshots in
+    unexpected ways and fails with an ``cannot encode tree at "[...]" without loosing information``
+    error otherwise. This can occur when rewriting a snapshot created by a newer
+    version of restic or some third-party implementation.
+
+    To convert a snapshot into the format expected by the ``rewrite`` command
+    use ``restic repair snapshots <snapshotID>``.
 
 Modifying metadata of snapshots
 ===============================

doc/050_restore.rst

@@ -132,6 +132,10 @@ options will be deleted. For example, the command
 ``restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo --delete``
 would only delete files within ``/tmp/restore-work/foo``.
 
+When using ``--target / --delete`` then the ``restore`` command only works if either an ``--include``
+or ``--exclude`` option is also specified. This ensures that one cannot accidentaly delete
+the whole system.
+
 Dry run
 -------
 

doc/075_scripting.rst

@@ -39,6 +39,8 @@ Note that restic will also return exit code ``1`` if a different error is encoun
 If there are no errors, restic will return a zero exit code and print the repository
 metadata.
 
+.. _exit-codes:
+
 Exit codes
 **********
 
@@ -63,6 +65,8 @@ a more specific description.
 +-----+----------------------------------------------------+
 | 11  | Failed to lock repository (since restic 0.17.0)    |
 +-----+----------------------------------------------------+
+| 12  | Wrong password (since restic 0.17.1)               |
++-----+----------------------------------------------------+
 | 130 | Restic was interrupted using SIGINT or SIGSTOP     |
 +-----+----------------------------------------------------+
 
@@ -139,7 +143,7 @@ Error
 +----------------------+-------------------------------------------+
 | ``message_type``     | Always "error"                            |
 +----------------------+-------------------------------------------+
-| ``error``            | Error message                             |
+| ``error.message``    | Error message                             |
 +----------------------+-------------------------------------------+
 | ``during``           | What restic was trying to do              |
 +----------------------+-------------------------------------------+
@@ -187,9 +191,9 @@ Summary is the last output line in a successful backup.
 +---------------------------+---------------------------------------------------------+
 | ``dirs_unmodified``       | Number of directories that did not change               |
 +---------------------------+---------------------------------------------------------+
-| ``data_blobs``            | Number of data blobs                                    |
+| ``data_blobs``            | Number of data blobs added                              |
 +---------------------------+---------------------------------------------------------+
-| ``tree_blobs``            | Number of tree blobs                                    |
+| ``tree_blobs``            | Number of tree blobs added                              |
 +---------------------------+---------------------------------------------------------+
 | ``data_added``            | Amount of (uncompressed) data added, in bytes           |
 +---------------------------+---------------------------------------------------------+
@@ -539,6 +543,19 @@ Status
 |``bytes_skipped``     | Total size of skipped files                                |
 +----------------------+------------------------------------------------------------+
 
+Error
+^^^^^
+
++----------------------+-------------------------------------------+
+| ``message_type``     | Always "error"                            |
++----------------------+-------------------------------------------+
+| ``error.message``    | Error message                             |
++----------------------+-------------------------------------------+
+| ``during``           | Always "restore"                          |
++----------------------+-------------------------------------------+
+| ``item``             | Usually, the path of the problematic file |
++----------------------+-------------------------------------------+
+
 Verbose Status
 ^^^^^^^^^^^^^^
 
@@ -634,9 +651,9 @@ was created.
 +---------------------------+---------------------------------------------------------+
 | ``dirs_unmodified``       | Number of directories that did not change               |
 +---------------------------+---------------------------------------------------------+
-| ``data_blobs``            | Number of data blobs                                    |
+| ``data_blobs``            | Number of data blobs added                              |
 +---------------------------+---------------------------------------------------------+
-| ``tree_blobs``            | Number of tree blobs                                    |
+| ``tree_blobs``            | Number of tree blobs added                              |
 +---------------------------+---------------------------------------------------------+
 | ``data_added``            | Amount of (uncompressed) data added, in bytes           |
 +---------------------------+---------------------------------------------------------+

doc/bash-completion.sh

@@ -516,6 +516,8 @@ _restic_backup()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -592,6 +594,8 @@ _restic_cache()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -660,6 +664,8 @@ _restic_cat()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -667,6 +673,15 @@ _restic_cat()
 
     must_have_one_flag=()
     must_have_one_noun=()
+    must_have_one_noun+=("blob")
+    must_have_one_noun+=("config")
+    must_have_one_noun+=("index")
+    must_have_one_noun+=("key")
+    must_have_one_noun+=("lock")
+    must_have_one_noun+=("masterkey")
+    must_have_one_noun+=("pack")
+    must_have_one_noun+=("snapshot")
+    must_have_one_noun+=("tree")
     noun_aliases=()
 }
 
@@ -736,6 +751,8 @@ _restic_check()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -840,6 +857,8 @@ _restic_copy()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -910,6 +929,8 @@ _restic_diff()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1004,6 +1025,78 @@ _restic_dump()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
+    flags+=("--tls-client-cert=")
+    two_word_flags+=("--tls-client-cert")
+    flags+=("--verbose")
+    flags+=("-v")
+
+    must_have_one_flag=()
+    must_have_one_noun=()
+    noun_aliases=()
+}
+
+_restic_features()
+{
+    last_command="restic_features"
+
+    command_aliases=()
+
+    commands=()
+
+    flags=()
+    two_word_flags=()
+    local_nonpersistent_flags=()
+    flags_with_completion=()
+    flags_completion=()
+
+    flags+=("--help")
+    flags+=("-h")
+    local_nonpersistent_flags+=("--help")
+    local_nonpersistent_flags+=("-h")
+    flags+=("--cacert=")
+    two_word_flags+=("--cacert")
+    flags+=("--cache-dir=")
+    two_word_flags+=("--cache-dir")
+    flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
+    flags+=("--http-user-agent=")
+    two_word_flags+=("--http-user-agent")
+    flags+=("--insecure-no-password")
+    flags+=("--insecure-tls")
+    flags+=("--json")
+    flags+=("--key-hint=")
+    two_word_flags+=("--key-hint")
+    flags+=("--limit-download=")
+    two_word_flags+=("--limit-download")
+    flags+=("--limit-upload=")
+    two_word_flags+=("--limit-upload")
+    flags+=("--no-cache")
+    flags+=("--no-extra-verify")
+    flags+=("--no-lock")
+    flags+=("--option=")
+    two_word_flags+=("--option")
+    two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
+    flags+=("--password-command=")
+    two_word_flags+=("--password-command")
+    flags+=("--password-file=")
+    two_word_flags+=("--password-file")
+    two_word_flags+=("-p")
+    flags+=("--quiet")
+    flags+=("-q")
+    flags+=("--repo=")
+    two_word_flags+=("--repo")
+    two_word_flags+=("-r")
+    flags+=("--repository-file=")
+    two_word_flags+=("--repository-file")
+    flags+=("--retry-lock=")
+    two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1122,6 +1215,8 @@ _restic_find()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1298,6 +1393,8 @@ _restic_forget()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1386,6 +1483,8 @@ _restic_generate()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1450,6 +1549,8 @@ _restic_help()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1547,6 +1648,8 @@ _restic_init()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1629,6 +1732,8 @@ _restic_key_add()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1693,6 +1798,8 @@ _restic_key_help()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1762,6 +1869,8 @@ _restic_key_list()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1844,6 +1953,8 @@ _restic_key_passwd()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1912,6 +2023,8 @@ _restic_key_remove()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -1985,6 +2098,8 @@ _restic_key()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2053,6 +2168,8 @@ _restic_list()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2060,6 +2177,12 @@ _restic_list()
 
     must_have_one_flag=()
     must_have_one_noun=()
+    must_have_one_noun+=("blobs")
+    must_have_one_noun+=("index")
+    must_have_one_noun+=("keys")
+    must_have_one_noun+=("locks")
+    must_have_one_noun+=("packs")
+    must_have_one_noun+=("snapshots")
     noun_aliases=()
 }
 
@@ -2145,6 +2268,8 @@ _restic_ls()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2217,6 +2342,8 @@ _restic_migrate()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2313,6 +2440,78 @@ _restic_mount()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
+    flags+=("--tls-client-cert=")
+    two_word_flags+=("--tls-client-cert")
+    flags+=("--verbose")
+    flags+=("-v")
+
+    must_have_one_flag=()
+    must_have_one_noun=()
+    noun_aliases=()
+}
+
+_restic_options()
+{
+    last_command="restic_options"
+
+    command_aliases=()
+
+    commands=()
+
+    flags=()
+    two_word_flags=()
+    local_nonpersistent_flags=()
+    flags_with_completion=()
+    flags_completion=()
+
+    flags+=("--help")
+    flags+=("-h")
+    local_nonpersistent_flags+=("--help")
+    local_nonpersistent_flags+=("-h")
+    flags+=("--cacert=")
+    two_word_flags+=("--cacert")
+    flags+=("--cache-dir=")
+    two_word_flags+=("--cache-dir")
+    flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
+    flags+=("--http-user-agent=")
+    two_word_flags+=("--http-user-agent")
+    flags+=("--insecure-no-password")
+    flags+=("--insecure-tls")
+    flags+=("--json")
+    flags+=("--key-hint=")
+    two_word_flags+=("--key-hint")
+    flags+=("--limit-download=")
+    two_word_flags+=("--limit-download")
+    flags+=("--limit-upload=")
+    two_word_flags+=("--limit-upload")
+    flags+=("--no-cache")
+    flags+=("--no-extra-verify")
+    flags+=("--no-lock")
+    flags+=("--option=")
+    two_word_flags+=("--option")
+    two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
+    flags+=("--password-command=")
+    two_word_flags+=("--password-command")
+    flags+=("--password-file=")
+    two_word_flags+=("--password-file")
+    two_word_flags+=("-p")
+    flags+=("--quiet")
+    flags+=("-q")
+    flags+=("--repo=")
+    two_word_flags+=("--repo")
+    two_word_flags+=("-r")
+    flags+=("--repository-file=")
+    two_word_flags+=("--repository-file")
+    flags+=("--retry-lock=")
+    two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2403,6 +2602,8 @@ _restic_prune()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2471,6 +2672,8 @@ _restic_recover()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2535,6 +2738,8 @@ _restic_repair_help()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2606,6 +2811,8 @@ _restic_repair_index()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2674,6 +2881,8 @@ _restic_repair_packs()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2762,6 +2971,8 @@ _restic_repair_snapshots()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2834,6 +3045,8 @@ _restic_repair()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -2970,6 +3183,8 @@ _restic_restore()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3084,6 +3299,8 @@ _restic_rewrite()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3156,6 +3373,8 @@ _restic_self-update()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3252,6 +3471,8 @@ _restic_snapshots()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3288,6 +3509,8 @@ _restic_stats()
     local_nonpersistent_flags+=("-H")
     flags+=("--mode=")
     two_word_flags+=("--mode")
+    flags_with_completion+=("--mode")
+    flags_completion+=("__restic_handle_go_custom_completion")
     local_nonpersistent_flags+=("--mode")
     local_nonpersistent_flags+=("--mode=")
     flags+=("--path=")
@@ -3338,6 +3561,8 @@ _restic_stats()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3432,6 +3657,8 @@ _restic_tag()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3502,6 +3729,8 @@ _restic_unlock()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3570,6 +3799,8 @@ _restic_version()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")
@@ -3594,6 +3825,7 @@ _restic_root_command()
     commands+=("copy")
     commands+=("diff")
     commands+=("dump")
+    commands+=("features")
     commands+=("find")
     commands+=("forget")
     commands+=("generate")
@@ -3604,6 +3836,7 @@ _restic_root_command()
     commands+=("ls")
     commands+=("migrate")
     commands+=("mount")
+    commands+=("options")
     commands+=("prune")
     commands+=("recover")
     commands+=("repair")
@@ -3666,6 +3899,8 @@ _restic_root_command()
     two_word_flags+=("--repository-file")
     flags+=("--retry-lock=")
     two_word_flags+=("--retry-lock")
+    flags+=("--stuck-request-timeout=")
+    two_word_flags+=("--stuck-request-timeout")
     flags+=("--tls-client-cert=")
     two_word_flags+=("--tls-client-cert")
     flags+=("--verbose")

doc/design.rst

@@ -126,8 +126,8 @@ the option ``-o local.layout=default``, valid values are ``default`` and
 ``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the
 s3 backend ``s3.layout``.
 
-S3 Legacy Layout
-----------------
+S3 Legacy Layout (deprecated)
+-----------------------------
 
 Unfortunately during development the Amazon S3 backend uses slightly different
 paths (directory names use singular instead of plural for ``key``,
@@ -152,8 +152,7 @@ the ``data`` directory. The S3 Legacy repository layout looks like this:
     /snapshot
      └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
 
-The S3 backend understands and accepts both forms, new backends are
-always created with the default layout for compatibility reasons.
+Restic 0.17 is the last version that supports the legacy layout.
 
 Pack Format
 ===========
@@ -234,7 +233,9 @@ Individual files for the index, locks or snapshots are encrypted
 and authenticated like Data and Tree Blobs, so the outer structure is
 ``IV || Ciphertext || MAC`` again. In repository format version 1 the
 plaintext always consists of a JSON document which must either be an
-object or an array.
+object or an array. The JSON encoder must deterministically encode the
+document and should match the behavior of the Go standard library implementation
+in ``encoding/json``.
 
 Repository format version 2 adds support for compression. The plaintext
 now starts with a header to indicate the encoding version to distinguish
@@ -473,6 +474,10 @@ A snapshot references a tree by the SHA-256 hash of the JSON string
 representation of its contents. Trees and data are saved in pack files
 in a subdirectory of the directory ``data``.
 
+The JSON encoder must deterministically encode the document and should
+match the behavior of the Go standard library implementation in ``encoding/json``.
+This ensures that trees can be properly deduplicated.
+
 The command ``restic cat blob`` can be used to inspect the tree
 referenced above (piping the output of the command to ``jq .`` so that
 the JSON is indented):
@@ -507,12 +512,11 @@ this metadata is generated:
 - The name is quoted using `strconv.Quote <https://pkg.go.dev/strconv#Quote>`__
   before being saved. This handles non-unicode names, but also changes the
   representation of names containing ``"`` or ``\``.
-
 - The filemode saved is the mode defined by `fs.FileMode <https://pkg.go.dev/io/fs#FileMode>`__
   masked by ``os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky``
-
-When the entry references a directory, the field ``subtree`` contains the plain text
-ID of another tree object.
+- When the entry references a directory, the field ``subtree`` contains the plain text
+  ID of another tree object.
+- Check the implementation for a full struct definition.
 
 When the command ``restic cat blob`` is used, the plaintext ID is needed
 to print a tree. The tree referenced above can be dumped as follows:

doc/faq.rst

@@ -90,7 +90,7 @@ The error here is that the tilde ``~`` in ``"~/documents"`` didn't get expanded
     /home/john/documents
 
     $ echo "~/documents"
-    ~/document
+    ~/documents
 
     $ echo "$HOME/documents"
     /home/john/documents
@@ -228,3 +228,17 @@ Restic backup command fails to find a valid file in Windows
 
 If the name of a file in Windows contains an invalid character, Restic will not be
 able to read the file. To solve this issue, consider renaming the particular file.
+
+What can I do in case of "request timeout" errors?
+--------------------------------------------------
+
+Restic monitors connections to the backend to detect stuck requests. If a request
+does not return any data within five minutes, restic assumes the request is stuck and
+retries it. However, for large repositories it sometimes takes longer than that to
+collect a list of all files, causing the following error:
+
+::
+
+    List(data) returned error, retrying after 1s: [...]: request timeout
+
+In this case you can increase the timeout using the ``--stuck-request-timeout`` option.

doc/man/restic-backup.1

@@ -24,6 +24,7 @@ Exit status is 1 if there was a fatal error (no snapshot created).
 Exit status is 3 if some source data could not be read (incomplete snapshot created).
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -229,6 +230,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-cache.1

@@ -129,6 +129,10 @@ Exit status is 1 if there was any error.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-cat.1

@@ -22,6 +22,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -119,6 +120,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-check.1

@@ -27,6 +27,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -136,6 +137,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-copy.1

@@ -36,12 +36,13 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
 .PP
 \fB--from-insecure-no-password\fP[=false]
-	use an empty password for the source repository, must be passed to every restic command (insecure)
+	use an empty password for the source repository (insecure)
 
 .PP
 \fB--from-key-hint\fP=""
@@ -169,6 +170,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-diff.1

@@ -49,6 +49,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -150,6 +151,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-dump.1

@@ -34,6 +34,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -151,6 +152,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-features.1

@@ -0,0 +1,146 @@
+.nh
+.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" ""
+
+.SH NAME
+.PP
+restic-features - Print list of feature flags
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic features [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "features" command prints a list of supported feature flags.
+
+.PP
+To pass feature flags to restic, set the RESTIC_FEATURES environment variable
+to "featureA=true,featureB=false". Specifying an unknown feature flag is an error.
+
+.PP
+A feature can either be in alpha, beta, stable or deprecated state.
+An \fIalpha\fP feature is disabled by default and may change in arbitrary ways between restic versions or be removed.
+A \fIbeta\fP feature is enabled by default, but still can change in minor ways or be removed.
+A \fIstable\fP feature is always enabled and cannot be disabled. The flag will be removed in a future restic version.
+A \fIdeprecated\fP feature is always disabled and cannot be enabled. The flag will be removed in a future restic version.
+
+
+.SH EXIT STATUS
+.PP
+Exit status is 0 if the command was successful.
+Exit status is 1 if there was any error.
+
+
+.SH OPTIONS
+.PP
+\fB-h\fP, \fB--help\fP[=false]
+	help for features
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB--cacert\fP=[]
+	\fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT)
+
+.PP
+\fB--cache-dir\fP=""
+	set the cache \fBdirectory\fR\&. (default: use system default cache directory)
+
+.PP
+\fB--cleanup-cache\fP[=false]
+	auto remove old cache directories
+
+.PP
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)
+
+.PP
+\fB--http-user-agent\fP=""
+	set a http user agent for outgoing http requests
+
+.PP
+\fB--insecure-no-password\fP[=false]
+	use an empty password for the repository, must be passed to every restic command (insecure)
+
+.PP
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
+	set output mode to JSON for commands that support it
+
+.PP
+\fB--key-hint\fP=""
+	\fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
+
+.PP
+\fB--limit-download\fP=0
+	limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited)
+
+.PP
+\fB--limit-upload\fP=0
+	limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited)
+
+.PP
+\fB--no-cache\fP[=false]
+	do not use a local cache
+
+.PP
+\fB--no-extra-verify\fP[=false]
+	skip additional verification of data before upload (see documentation)
+
+.PP
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
+
+.PP
+\fB-o\fP, \fB--option\fP=[]
+	set extended option (\fBkey=value\fR, can be specified multiple times)
+
+.PP
+\fB--pack-size\fP=0
+	set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+
+.PP
+\fB--password-command\fP=""
+	shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
+
+.PP
+\fB-p\fP, \fB--password-file\fP=""
+	\fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
+	do not output comprehensive progress report
+
+.PP
+\fB-r\fP, \fB--repo\fP=""
+	\fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
+
+.PP
+\fB--repository-file\fP=""
+	\fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
+
+.PP
+\fB--retry-lock\fP=0s
+	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
+
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
+.PP
+\fB--tls-client-cert\fP=""
+	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)
+
+.PP
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-find.1

@@ -165,6 +165,10 @@ It can also be used to search for restic blobs or trees for troubleshooting.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)
@@ -190,6 +194,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 .EE
 

doc/man/restic-forget.1

@@ -36,6 +36,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -237,6 +238,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-generate.1

@@ -138,6 +138,10 @@ Exit status is 1 if there was any error.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-init.1

@@ -29,7 +29,7 @@ Exit status is 1 if there was any error.
 
 .PP
 \fB--from-insecure-no-password\fP[=false]
-	use an empty password for the source repository, must be passed to every restic command (insecure)
+	use an empty password for the source repository (insecure)
 
 .PP
 \fB--from-key-hint\fP=""
@@ -149,6 +149,10 @@ Exit status is 1 if there was any error.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-key-add.1

@@ -22,6 +22,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -135,6 +136,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)

doc/man/restic-key-list.1

@@ -24,6 +24,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
+Exit status is 12 if the password is incorrect.
 
 
 .SH OPTIONS
@@ -121,6 +122,10 @@ Exit status is 11 if the repository is already locked.
 \fB--retry-lock\fP=0s
 	retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)
 
+.PP
+\fB--stuck-request-timeout\fP=5m0s
+	\fBduration\fR after which to retry stuck requests
+
 .PP
 \fB--tls-client-cert\fP=""
 	path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)