wip

.github/workflows/docker.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+        uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -33,7 +33,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -45,7 +45,7 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
 
       - name: Ensure consistent binaries
         run: |
@@ -55,7 +55,7 @@ jobs:
         if: github.ref != 'refs/heads/master'
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
           push: true
           context: .

.github/workflows/tests.yml

@@ -74,7 +74,7 @@ jobs:
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go install github.com/restic/rest-server/cmd/rest-server@master
+          go install github.com/restic/rest-server/cmd/rest-server@latest
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -106,7 +106,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go install github.com/restic/rest-server/cmd/rest-server@master
+          go install github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -247,10 +247,6 @@ jobs:
   lint:
     name: lint
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      # allow annotating code in the PR
-      checks: write
     steps:
       - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v5
@@ -261,10 +257,10 @@ jobs:
         uses: actions/checkout@v4
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.57.1
+          version: v1.56.1
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
@@ -302,7 +298,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -325,7 +321,7 @@ jobs:
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v4
         with:
           push: false
           context: .

.golangci.yml

@@ -38,8 +38,6 @@ linters:
     # ensure that http response bodies are closed
     - bodyclose
 
-    - importas
-
 issues:
   # don't use the default exclude rules, this hides (among others) ignored
   # errors from Close() calls
@@ -61,9 +59,3 @@ issues:
     # revive: ignore unused parameters in tests
     - path: (_test\.go|testing\.go|backend/.*/tests\.go)
       text: "unused-parameter:"
-
-linters-settings:
-  importas:
-    alias:
-      - pkg: github.com/restic/restic/internal/test
-        alias: rtest

CHANGELOG.md

@@ -1,7 +1,5 @@
 # Table of Contents
 
-* [Changelog for 0.17.0](#changelog-for-restic-0170-2024-07-26)
-* [Changelog for 0.16.5](#changelog-for-restic-0165-2024-07-01)
 * [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04)
 * [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14)
 * [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29)
@@ -35,738 +33,6 @@
 * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
 
 
-# Changelog for restic 0.17.0 (2024-07-26)
-The following sections list the changes in restic 0.17.0 relevant to
-restic users. The changes are ordered by importance.
-
-## Summary
-
- * Fix #3600: Handle unreadable xattrs in folders above `backup` source
- * Fix #4209: Fix slow SFTP upload performance
- * Fix #4503: Correct hardlink handling in `stats` command
- * Fix #4568: Prevent `forget --keep-tags <invalid>` from deleting all snapshots
- * Fix #4615: Make `find` not sometimes ignore directories
- * Fix #4656: Properly report ID of newly added keys
- * Fix #4703: Shutdown cleanly when receiving SIGTERM
- * Fix #4709: Correct `--no-lock` handling of `ls` and `tag` commands
- * Fix #4760: Fix possible error on concurrent cache cleanup
- * Fix #4850: Handle UTF-16 password files in `key` command correctly
- * Fix #4902: Update snapshot summary on `rewrite`
- * Chg #956: Return exit code 10 and 11 for non-existing and locked repository
- * Chg #4540: Require at least ARMv6 for ARM binaries
- * Chg #4602: Deprecate legacy index format and `s3legacy` repository layout
- * Chg #4627: Redesign backend error handling to improve reliability
- * Chg #4707: Disable S3 anonymous authentication by default
- * Chg #4744: Include full key ID in JSON output of `key list`
- * Enh #662: Optionally skip snapshot creation if nothing changed
- * Enh #693: Include snapshot size in `snapshots` output
- * Enh #805: Add bitrot detection to `diff` command
- * Enh #828: Improve features of the `repair packs` command
- * Enh #1786: Support repositories with empty password
- * Enh #2348: Add `--delete` option to `restore` command
- * Enh #3067: Add extended options to configure Windows Shadow Copy Service
- * Enh #3406: Improve `dump` performance for large files
- * Enh #3806: Optimize and make `prune` command resumable
- * Enh #4006: (alpha) Store deviceID only for hardlinks
- * Enh #4048: Add support for FUSE-T with `mount` on macOS
- * Enh #4251: Support reading backup from a command's standard output
- * Enh #4287: Support connection to rest-server using unix socket
- * Enh #4354: Significantly reduce `prune` memory usage
- * Enh #4437: Make `check` command create non-existent cache directory
- * Enh #4472: Support AWS Assume Role for S3 backend
- * Enh #4547: Add `--json` option to `version` command
- * Enh #4549: Add `--ncdu` option to `ls` command
- * Enh #4573: Support rewriting host and time metadata in snapshots
- * Enh #4583: Ignore `s3.storage-class` archive tiers for metadata
- * Enh #4590: Speed up `mount` command's error detection
- * Enh #4601: Add support for feature flags
- * Enh #4611: Back up more file metadata on Windows
- * Enh #4664: Make `ls` use `message_type` field in JSON output
- * Enh #4676: Make `key` command's actions separate sub-commands
- * Enh #4678: Add `--target` option to the `dump` command
- * Enh #4708: Back up and restore SecurityDescriptors on Windows
- * Enh #4733: Allow specifying `--host` via environment variable
- * Enh #4737: Include snapshot ID in `reason` field of `forget` JSON output
- * Enh #4764: Support forgetting all snapshots
- * Enh #4768: Allow specifying custom User-Agent for outgoing requests
- * Enh #4781: Add `restore` options to read include/exclude patterns from files
- * Enh #4807: Support Extended Attributes on Windows NTFS
- * Enh #4817: Make overwrite behavior of `restore` customizable
- * Enh #4839: Add dry-run support to `restore` command
-
-## Details
-
- * Bugfix #3600: Handle unreadable xattrs in folders above `backup` source
-
-   When backup sources are specified using absolute paths, `backup` also includes
-   information about the parent folders of the backup sources in the snapshot.
-
-   If the extended attributes for some of these folders could not be read due to
-   missing permissions, this caused the backup to fail. This has now been fixed.
-
-   https://github.com/restic/restic/issues/3600
-   https://github.com/restic/restic/pull/4668
-   https://forum.restic.net/t/parent-directories-above-the-snapshot-source-path-fatal-error-permission-denied/7216
-
- * Bugfix #4209: Fix slow SFTP upload performance
-
-   Since restic 0.12.1, the upload speed of the sftp backend to a remote server has
-   regressed significantly. This has now been fixed.
-
-   https://github.com/restic/restic/issues/4209
-   https://github.com/restic/restic/pull/4782
-
- * Bugfix #4503: Correct hardlink handling in `stats` command
-
-   If files on different devices had the same inode ID, the `stats` command did not
-   correctly calculate the snapshot size. This has now been fixed.
-
-   https://github.com/restic/restic/pull/4503
-   https://github.com/restic/restic/pull/4006
-   https://forum.restic.net/t/possible-bug-in-stats/6461/8
-
- * Bugfix #4568: Prevent `forget --keep-tags <invalid>` from deleting all snapshots
-
-   Running `forget --keep-tags <invalid>`, where `<invalid>` is a tag that does not
-   exist in the repository, would remove all snapshots. This is especially
-   problematic if the tag name contains a typo.
-
-   The `forget` command now fails with an error if all snapshots in a snapshot
-   group would be deleted. This prevents the above example from deleting all
-   snapshots.
-
-   It is possible to temporarily disable the new check by setting the environment
-   variable `RESTIC_FEATURES=safe-forget-keep-tags=false`. Note that this feature
-   flag will be removed in the next minor restic version.
-
-   https://github.com/restic/restic/pull/4568
-   https://github.com/restic/restic/pull/4764
-
- * Bugfix #4615: Make `find` not sometimes ignore directories
-
-   In some cases, the `find` command ignored empty or moved directories. This has
-   now been fixed.
-
-   https://github.com/restic/restic/pull/4615
-
- * Bugfix #4656: Properly report ID of newly added keys
-
-   `restic key add` now reports the ID of the newly added key. This simplifies
-   selecting a specific key using the `--key-hint key` option.
-
-   https://github.com/restic/restic/issues/4656
-   https://github.com/restic/restic/pull/4657
-
- * Bugfix #4703: Shutdown cleanly when receiving SIGTERM
-
-   Previously, when restic received the SIGTERM signal it would terminate
-   immediately, skipping cleanup and potentially causing issues like stale locks
-   being left behind. This primarily effected containerized restic invocations that
-   use SIGTERM, but could also be triggered via a simple `killall restic`.
-
-   This has now been fixed, such that restic shuts down cleanly when receiving the
-   SIGTERM signal.
-
-   https://github.com/restic/restic/pull/4703
-
- * Bugfix #4709: Correct `--no-lock` handling of `ls` and `tag` commands
-
-   The `ls` command never locked the repository. This has now been fixed, with the
-   old behavior still being supported using `ls --no-lock`. The latter invocation
-   also works with older restic versions.
-
-   The `tag` command erroneously accepted the `--no-lock` command. This command now
-   always requires an exclusive lock.
-
-   https://github.com/restic/restic/pull/4709
-
- * Bugfix #4760: Fix possible error on concurrent cache cleanup
-
-   If multiple restic processes concurrently cleaned up no longer existing files
-   from the cache, this could cause some of the processes to fail with an `no such
-   file or directory` error. This has now been fixed.
-
-   https://github.com/restic/restic/issues/4760
-   https://github.com/restic/restic/pull/4761
-
- * Bugfix #4850: Handle UTF-16 password files in `key` command correctly
-
-   Previously, `key add` and `key passwd` did not properly decode UTF-16 encoded
-   passwords read from a password file. This has now been fixed to correctly match
-   the encoding when opening a repository.
-
-   https://github.com/restic/restic/issues/4850
-   https://github.com/restic/restic/pull/4851
-
- * Bugfix #4902: Update snapshot summary on `rewrite`
-
-   Restic previously did not recalculate the total number of files and bytes
-   processed when files were excluded from a snapshot by the `rewrite` command.
-   This has now been fixed.
-
-   https://github.com/restic/restic/issues/4902
-   https://github.com/restic/restic/pull/4905
-
- * Change #956: Return exit code 10 and 11 for non-existing and locked repository
-
-   If a repository does not exist or cannot be locked, restic previously always
-   returned exit code 1. This made it difficult to distinguish these cases from
-   other errors.
-
-   Restic now returns exit code 10 if the repository does not exist, and exit code
-   11 if the repository could be not locked due to a conflicting lock.
-
-   https://github.com/restic/restic/issues/956
-   https://github.com/restic/restic/pull/4884
-
- * Change #4540: Require at least ARMv6 for ARM binaries
-
-   The official release binaries of restic now require at least ARMv6 support for
-   ARM platforms.
-
-   https://github.com/restic/restic/issues/4540
-   https://github.com/restic/restic/pull/4542
-
- * Change #4602: Deprecate legacy index format and `s3legacy` repository layout
-
-   Support for the legacy index format used by restic before version 0.2.0 has been
-   deprecated and will be removed in the next minor restic version. You can use
-   `restic repair index` to update the index to the current format.
-
-   It is possible to temporarily reenable support for the legacy index format by
-   setting the environment variable `RESTIC_FEATURES=deprecate-legacy-index=false`.
-   Note that this feature flag will be removed in the next minor restic version.
-
-   Support for the `s3legacy` repository layout used for the S3 backend before
-   restic 0.7.0 has been deprecated and will be removed in the next minor restic
-   version. You can migrate your S3 repository to the current layout using
-   `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout`.
-
-   It is possible to temporarily reenable support for the `s3legacy` layout by
-   setting the environment variable
-   `RESTIC_FEATURES=deprecate-s3-legacy-layout=false`. Note that this feature flag
-   will be removed in the next minor restic version.
-
-   https://github.com/restic/restic/issues/4602
-   https://github.com/restic/restic/pull/4724
-   https://github.com/restic/restic/pull/4743
-
- * Change #4627: Redesign backend error handling to improve reliability
-
-   Restic now downloads pack files in large chunks instead of using a streaming
-   download. This prevents failures due to interrupted streams. The `restore`
-   command now also retries downloading individual blobs that could not be
-   retrieved.
-
-   HTTP requests that are stuck for more than two minutes while uploading or
-   downloading are now forcibly interrupted. This ensures that stuck requests are
-   retried after a short timeout.
-
-   Attempts to access a missing or truncated file will no longer be retried. This
-   avoids unnecessary retries in those cases. All other backend requests are
-   retried for up to 15 minutes. This ensures that temporarily interrupted network
-   connections can be tolerated.
-
-   If a download yields a corrupt file or blob, then the download will be retried
-   once.
-
-   Most parts of the new backend error handling can temporarily be disabled by
-   setting the environment variable `RESTIC_FEATURES=backend-error-redesign=false`.
-   Note that this feature flag will be removed in the next minor restic version.
-
-   https://github.com/restic/restic/issues/4627
-   https://github.com/restic/restic/issues/4193
-   https://github.com/restic/restic/issues/4515
-   https://github.com/restic/restic/issues/1523
-   https://github.com/restic/restic/pull/4605
-   https://github.com/restic/restic/pull/4792
-   https://github.com/restic/restic/pull/4520
-   https://github.com/restic/restic/pull/4800
-   https://github.com/restic/restic/pull/4784
-   https://github.com/restic/restic/pull/4844
-
- * Change #4707: Disable S3 anonymous authentication by default
-
-   When using the S3 backend with anonymous authentication, it continuously tried
-   to retrieve new authentication credentials, causing bad performance.
-
-   Now, to use anonymous authentication, it is necessary to pass the extended
-   option `-o s3.unsafe-anonymous-auth=true` to restic.
-
-   It is possible to temporarily revert to the old behavior by setting the
-   environment variable `RESTIC_FEATURES=explicit-s3-anonymous-auth=false`. Note
-   that this feature flag will be removed in the next minor restic version.
-
-   https://github.com/restic/restic/issues/4707
-   https://github.com/restic/restic/pull/4908
-
- * Change #4744: Include full key ID in JSON output of `key list`
-
-   The JSON output of the `key list` command has changed to include the full key ID
-   instead of just a shortened version of the ID, as the latter can be ambiguous in
-   some rare cases. To derive the short ID, please truncate the full ID down to
-   eight characters.
-
-   https://github.com/restic/restic/issues/4744
-   https://github.com/restic/restic/pull/4745
-
- * Enhancement #662: Optionally skip snapshot creation if nothing changed
-
-   The `backup` command always created a snapshot even if nothing in the backup set
-   changed compared to the parent snapshot.
-
-   Restic now supports the `--skip-if-unchanged` option for the `backup` command,
-   which omits creating a snapshot if the new snapshot's content would be identical
-   to that of the parent snapshot.
-
-   https://github.com/restic/restic/issues/662
-   https://github.com/restic/restic/pull/4816
-
- * Enhancement #693: Include snapshot size in `snapshots` output
-
-   The `snapshots` command now prints the size for snapshots created using this or
-   a future restic version. To achieve this, the `backup` command now stores the
-   backup summary statistics in the snapshot.
-
-   The text output of the `snapshots` command only shows the snapshot size. The
-   other statistics are only included in the JSON output. To inspect these
-   statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
-
-   https://github.com/restic/restic/issues/693
-   https://github.com/restic/restic/pull/4705
-   https://github.com/restic/restic/pull/4913
-
- * Enhancement #805: Add bitrot detection to `diff` command
-
-   The output of the `diff` command now includes the modifier `?` for files to
-   indicate bitrot in backed up files. The `?` will appear whenever there is a
-   difference in content while the metadata is exactly the same.
-
-   Since files with unchanged metadata are normally not read again when creating a
-   backup, the detection is only effective when the right-hand side of the diff has
-   been created with `backup --force`.
-
-   https://github.com/restic/restic/issues/805
-   https://github.com/restic/restic/pull/4526
-
- * Enhancement #828: Improve features of the `repair packs` command
-
-   The `repair packs` command has been improved to also be able to process
-   truncated pack files. The `check` and `check --read-data` command will provide
-   instructions on using the command if necessary to repair a repository. See the
-   guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html for
-   further instructions.
-
-   https://github.com/restic/restic/issues/828
-   https://github.com/restic/restic/pull/4644
-   https://github.com/restic/restic/pull/4882
-
- * Enhancement #1786: Support repositories with empty password
-
-   Restic previously required a password to create or operate on repositories.
-   Using the new option `--insecure-no-password` it is now possible to disable this
-   requirement. Restic will not prompt for a password when using this option.
-
-   For security reasons, the option must always be specified when operating on
-   repositories with an empty password, and specifying `--insecure-no-password`
-   while also passing a password to restic via a CLI option or environment variable
-   results in an error.
-
-   The `init` and `copy` commands add the related `--from-insecure-no-password`
-   option, which applies to the source repository. The `key add` and `key passwd`
-   commands add the `--new-insecure-no-password` option to add or set an empty
-   password.
-
-   https://github.com/restic/restic/issues/1786
-   https://github.com/restic/restic/issues/4326
-   https://github.com/restic/restic/pull/4698
-   https://github.com/restic/restic/pull/4808
-
- * Enhancement #2348: Add `--delete` option to `restore` command
-
-   The `restore` command now supports a `--delete` option that allows removing
-   files and directories from the target directory that do not exist in the
-   snapshot. This option also allows files in the snapshot to replace non-empty
-   directories having the same name.
-
-   To check that only expected files are deleted, add the `--dry-run --verbose=2`
-   options.
-
-   https://github.com/restic/restic/issues/2348
-   https://github.com/restic/restic/pull/4881
-
- * Enhancement #3067: Add extended options to configure Windows Shadow Copy Service
-
-   Previous, restic always used a 120 seconds timeout and unconditionally created
-   VSS snapshots for all volume mount points on disk. This behavior can now be
-   fine-tuned by the following new extended options (available only on Windows):
-
-   - `-o vss.timeout`: Time that VSS can spend creating snapshot before timing out
-   (default: 120s) - `-o vss.exclude-all-mount-points`: Exclude mountpoints from
-   snapshotting on all volumes (default: false) - `-o vss.exclude-volumes`:
-   Semicolon separated list of volumes to exclude from snapshotting - `-o
-   vss.provider`: VSS provider identifier which will be used for snapshotting
-
-   For example, change VSS timeout to five minutes and disable snapshotting of
-   mount points on all volumes:
-
-   Restic backup --use-fs-snapshot -o vss.timeout=5m -o
-   vss.exclude-all-mount-points=true
-
-   Exclude drive `d:`, mount point `c:\mnt` and a specific volume from
-   snapshotting:
-
-   Restic backup --use-fs-snapshot -o
-   vss.exclude-volumes="d:\;c:\mnt\;\\?\Volume{e2e0315d-9066-4f97-8343-eb5659b35762}"
-
-   Uses 'Microsoft Software Shadow Copy provider 1.0' instead of the default
-   provider:
-
-   Restic backup --use-fs-snapshot -o
-   vss.provider={b5946137-7b9f-4925-af80-51abd60b20d5}
-
-   https://github.com/restic/restic/pull/3067
-
- * Enhancement #3406: Improve `dump` performance for large files
-
-   The `dump` command now retrieves the data chunks for a file in parallel. This
-   improves the download performance by up to as many times as the configured
-   number of parallel backend connections.
-
-   https://github.com/restic/restic/issues/3406
-   https://github.com/restic/restic/pull/4796
-
- * Enhancement #3806: Optimize and make `prune` command resumable
-
-   Previously, if the `prune` command was interrupted, a later `prune` run would
-   start repacking pack files from the start, as `prune` did not update the index
-   while repacking.
-
-   The `prune` command now supports resuming interrupted prune runs. The update of
-   the repository index has also been optimized to use less memory and only rewrite
-   parts of the index that have changed.
-
-   https://github.com/restic/restic/issues/3806
-   https://github.com/restic/restic/pull/4812
-
- * Enhancement #4006: (alpha) Store deviceID only for hardlinks
-
-   Set `RESTIC_FEATURES=device-id-for-hardlinks` to enable this alpha feature. The
-   feature flag will be removed after repository format version 3 becomes available
-   or be replaced with a different solution.
-
-   When creating backups from a filesystem snapshot, for example created using
-   BTRFS subvolumes, the deviceID of the filesystem changes compared to previous
-   snapshots. This prevented restic from deduplicating the directory metadata of a
-   snapshot.
-
-   When this alpha feature is enabled, the deviceID is only stored for hardlinks,
-   which significantly reduces the metadata duplication for most backups.
-
-   https://github.com/restic/restic/pull/4006
-
- * Enhancement #4048: Add support for FUSE-T with `mount` on macOS
-
-   The restic `mount` command now supports creating FUSE mounts using FUSE-T on
-   macOS.
-
-   https://github.com/restic/restic/issues/4048
-   https://github.com/restic/restic/pull/4825
-
- * Enhancement #4251: Support reading backup from a command's standard output
-
-   The `backup` command now supports the `--stdin-from-command` option. When using
-   this option, the arguments to `backup` are interpreted as a command instead of
-   paths to back up. `backup` then executes the given command and stores the
-   standard output from it in the backup, similar to the what the `--stdin` option
-   does. This also enables restic to verify that the command completes with exit
-   code zero. A non-zero exit code causes the backup to fail.
-
-   Note that the `--stdin` option does not have to be specified at the same time,
-   and that the `--stdin-filename` option also applies to `--stdin-from-command`.
-
-   Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump
-   [...]`
-
-   https://github.com/restic/restic/issues/4251
-   https://github.com/restic/restic/pull/4410
-
- * Enhancement #4287: Support connection to rest-server using unix socket
-
-   Restic now supports using a unix socket to connect to a rest-server version
-   0.13.0 or later. This allows running restic as follows:
-
-   ```
-   rest-server --listen unix:/tmp/rest.socket --data /path/to/data &
-   restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...]
-   ```
-
-   https://github.com/restic/restic/issues/4287
-   https://github.com/restic/restic/pull/4655
-
- * Enhancement #4354: Significantly reduce `prune` memory usage
-
-   The `prune` command has been optimized to use up to 60% less memory. The memory
-   usage should now be roughly similar to creating a backup.
-
-   https://github.com/restic/restic/pull/4354
-   https://github.com/restic/restic/pull/4812
-
- * Enhancement #4437: Make `check` command create non-existent cache directory
-
-   Previously, if a custom cache directory was specified for the `check` command,
-   but the directory did not exist, `check` continued with the cache disabled.
-
-   The `check` command now attempts to create the cache directory before
-   initializing the cache.
-
-   https://github.com/restic/restic/issues/4437
-   https://github.com/restic/restic/pull/4805
-   https://github.com/restic/restic/pull/4883
-
- * Enhancement #4472: Support AWS Assume Role for S3 backend
-
-   Previously only credentials discovered via the Minio discovery methods were used
-   to authenticate.
-
-   However, there are many circumstances where the discovered credentials have
-   lower permissions and need to assume a specific role. This is now possible using
-   the following new environment variables:
-
-   - RESTIC_AWS_ASSUME_ROLE_ARN - RESTIC_AWS_ASSUME_ROLE_SESSION_NAME -
-   RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID - RESTIC_AWS_ASSUME_ROLE_REGION (defaults to
-   us-east-1) - RESTIC_AWS_ASSUME_ROLE_POLICY - RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT
-
-   https://github.com/restic/restic/issues/4472
-   https://github.com/restic/restic/pull/4474
-
- * Enhancement #4547: Add `--json` option to `version` command
-
-   Restic now supports outputting restic version along with the Go version, OS and
-   architecture used to build restic in JSON format using `version --json`.
-
-   https://github.com/restic/restic/issues/4547
-   https://github.com/restic/restic/pull/4553
-
- * Enhancement #4549: Add `--ncdu` option to `ls` command
-
-   NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. It has
-   an option to save a directory tree and analyse it later.
-
-   The `ls` command now supports outputting snapshot information in the NCDU format
-   using the `--ncdu` option. Example usage: `restic ls latest --ncdu | ncdu -f -`
-
-   https://github.com/restic/restic/issues/4549
-   https://github.com/restic/restic/pull/4550
-   https://github.com/restic/restic/pull/4911
-
- * Enhancement #4573: Support rewriting host and time metadata in snapshots
-
-   The `rewrite` command now supports rewriting the host and/or time metadata of a
-   snapshot using the new `--new-host` and `--new-time` options.
-
-   https://github.com/restic/restic/pull/4573
-
- * Enhancement #4583: Ignore `s3.storage-class` archive tiers for metadata
-
-   Restic used to store all files on S3 using the specified `s3.storage-class`.
-
-   Now, restic will only use non-archive storage tiers for metadata, to avoid
-   problems when accessing a repository. To restore any data, it is still necessary
-   to manually warm up the required data beforehand.
-
-   NOTE: There is no official cold storage support in restic, use this option at
-   your own risk.
-
-   https://github.com/restic/restic/issues/4583
-   https://github.com/restic/restic/pull/4584
-
- * Enhancement #4590: Speed up `mount` command's error detection
-
-   The `mount` command now checks for the existence of the mountpoint before
-   opening the repository, leading to quicker error detection.
-
-   https://github.com/restic/restic/pull/4590
-
- * Enhancement #4601: Add support for feature flags
-
-   Restic now supports feature flags that can be used to enable and disable
-   experimental features. The flags can be set using the environment variable
-   `RESTIC_FEATURES`. To get a list of currently supported feature flags, use the
-   `features` command.
-
-   https://github.com/restic/restic/issues/4601
-   https://github.com/restic/restic/pull/4666
-
- * Enhancement #4611: Back up more file metadata on Windows
-
-   Previously, restic did not back up all common Windows-specific metadata.
-
-   Restic now stores file creation time and file attributes like the hidden,
-   read-only and encrypted flags when backing up files and folders on Windows.
-
-   https://github.com/restic/restic/pull/4611
-
- * Enhancement #4664: Make `ls` use `message_type` field in JSON output
-
-   The `ls` command was the only restic command that used the `struct_type` field
-   in its JSON output format to specify the message type.
-
-   The JSON output of the `ls` command now also includes the `message_type` field,
-   which is consistent with other commands. The `struct_type` field is still
-   included, but now deprecated.
-
-   https://github.com/restic/restic/pull/4664
-
- * Enhancement #4676: Make `key` command's actions separate sub-commands
-
-   Each of the `add`, `list`, `remove` and `passwd` actions provided by the `key`
-   command is now a separate sub-command and have its own documentation which can
-   be invoked using `restic key <add|list|remove|passwd> --help`.
-
-   https://github.com/restic/restic/issues/4676
-   https://github.com/restic/restic/pull/4685
-
- * Enhancement #4678: Add `--target` option to the `dump` command
-
-   Restic `dump` always printed to the standard output. It now supports specifying
-   a `--target` file to write its output to.
-
-   https://github.com/restic/restic/issues/4678
-   https://github.com/restic/restic/pull/4682
-   https://github.com/restic/restic/pull/4692
-
- * Enhancement #4708: Back up and restore SecurityDescriptors on Windows
-
-   Restic now backs up and restores SecurityDescriptors for files and folders on
-   Windows which includes owner, group, discretionary access control list (DACL)
-   and system access control list (SACL).
-
-   This requires the user to be a member of backup operators or the application
-   must be run as admin. If that is not the case, only the current user's owner,
-   group and DACL will be backed up, and during restore only the DACL of the backed
-   up file will be restored, with the current user's owner and group being set on
-   the restored file.
-
-   https://github.com/restic/restic/pull/4708
-
- * Enhancement #4733: Allow specifying `--host` via environment variable
-
-   Restic commands that operate on snapshots, such as `restic backup` and `restic
-   snapshots`, support the `--host` option to specify the hostname for grouping
-   snapshots.
-
-   Such commands now also support specifying the hostname via the environment
-   variable `RESTIC_HOST`. Note that `--host` still takes precedence over the
-   environment variable.
-
-   https://github.com/restic/restic/issues/4733
-   https://github.com/restic/restic/pull/4734
-
- * Enhancement #4737: Include snapshot ID in `reason` field of `forget` JSON output
-
-   The JSON output of the `forget` command now includes `id` and `short_id` of
-   snapshots in the `reason` field.
-
-   https://github.com/restic/restic/pull/4737
-
- * Enhancement #4764: Support forgetting all snapshots
-
-   The `forget` command now supports the `--unsafe-allow-remove-all` option, which
-   removes all snapshots in the repository.
-
-   This option must always be combined with a snapshot filter (by host, path or
-   tag). For example, the command `forget --tag example --unsafe-allow-remove-all`
-   removes all snapshots with the tag "example".
-
-   https://github.com/restic/restic/pull/4764
-
- * Enhancement #4768: Allow specifying custom User-Agent for outgoing requests
-
-   Restic now supports setting a custom `User-Agent` for outgoing HTTP requests
-   using the global option `--http-user-agent` or the `RESTIC_HTTP_USER_AGENT`
-   environment variable.
-
-   https://github.com/restic/restic/issues/4768
-   https://github.com/restic/restic/pull/4810
-
- * Enhancement #4781: Add `restore` options to read include/exclude patterns from files
-
-   Restic now supports reading include and exclude patterns from files using the
-   `--include-file`, `--exclude-file`, `--iinclude-file` and `--iexclude-file`
-   options of the `restore` command.
-
-   https://github.com/restic/restic/issues/4781
-   https://github.com/restic/restic/pull/4811
-
- * Enhancement #4807: Support Extended Attributes on Windows NTFS
-
-   Restic now backs up and restores Extended Attributes for files and folders on
-   Windows NTFS.
-
-   https://github.com/restic/restic/pull/4807
-
- * Enhancement #4817: Make overwrite behavior of `restore` customizable
-
-   The `restore` command now supports an `--overwrite` option to configure whether
-   already existing files are overwritten. The overwrite behavior can be configured
-   using the following option values:
-
-   - `--overwrite always` (default): Always overwrites already existing files. The
-   `restore` command will verify the existing file content and only restore
-   mismatching parts to minimize downloads. Updates the metadata of all files. -
-   `--overwrite if-changed`: Like `always`, but speeds up the file content check by
-   assuming that files with matching size and modification time (mtime) are already
-   up to date. In case of a mismatch, the full file content is verified like with
-   `always`. Updates the metadata of all files. - `--overwrite if-newer`: Like
-   `always`, but only overwrites existing files when the file in the snapshot has a
-   newer modification time (mtime) than the existing file. - `--overwrite never`:
-   Never overwrites existing files.
-
-   https://github.com/restic/restic/issues/4817
-   https://github.com/restic/restic/issues/200
-   https://github.com/restic/restic/issues/407
-   https://github.com/restic/restic/issues/2662
-   https://github.com/restic/restic/pull/4837
-   https://github.com/restic/restic/pull/4838
-   https://github.com/restic/restic/pull/4864
-   https://github.com/restic/restic/pull/4921
-
- * Enhancement #4839: Add dry-run support to `restore` command
-
-   The `restore` command now supports the `--dry-run` option to perform a dry run.
-   Pass the `--verbose=2` option to see which files would remain unchanged, and
-   which would be updated or freshly restored.
-
-   https://github.com/restic/restic/pull/4839
-
-
-# Changelog for restic 0.16.5 (2024-07-01)
-The following sections list the changes in restic 0.16.5 relevant to
-restic users. The changes are ordered by importance.
-
-## Summary
-
- * Enh #4799: Add option to force use of Azure CLI credential
- * Enh #4873: Update dependencies
-
-## Details
-
- * Enhancement #4799: Add option to force use of Azure CLI credential
-
-   A new environment variable `AZURE_FORCE_CLI_CREDENTIAL=true` allows forcing the
-   use of Azure CLI credential, ignoring other credentials like managed identity.
-
-   https://github.com/restic/restic/pull/4799
-
- * Enhancement #4873: Update dependencies
-
-   A few potentially vulnerable dependencies were updated.
-
-   https://github.com/restic/restic/issues/4873
-   https://github.com/restic/restic/pull/4878
-
-
 # Changelog for restic 0.16.4 (2024-02-04)
 The following sections list the changes in restic 0.16.4 relevant to
 restic users. The changes are ordered by importance.
@@ -1330,7 +596,7 @@ restic users. The changes are ordered by importance.
  * Enhancement #3941: Support `--group-by` for backup parent selection
 
    Previously, the `backup` command by default selected the parent snapshot based
-   on the hostname and the backup paths. When the backup path list changed, the
+   on the hostname and the backup targets. When the backup path list changed, the
    `backup` command was unable to determine a suitable parent snapshot and had to
    read all files again.
 
@@ -3224,7 +2490,7 @@ restic users. The changes are ordered by importance.
  * Fix #3151: Don't create invalid snapshots when `backup` is interrupted
  * Fix #3152: Do not hang until foregrounded when completed in background
  * Fix #3166: Improve error handling in the `restore` command
- * Fix #3232: Correct statistics for overlapping backup sources
+ * Fix #3232: Correct statistics for overlapping targets
  * Fix #3249: Improve error handling in `gs` backend
  * Chg #3095: Deleting files on Google Drive now moves them to the trash
  * Enh #909: Back up mountpoints as empty directories
@@ -3391,10 +2657,10 @@ restic users. The changes are ordered by importance.
    https://github.com/restic/restic/issues/3166
    https://github.com/restic/restic/pull/3207
 
- * Bugfix #3232: Correct statistics for overlapping backup sources
+ * Bugfix #3232: Correct statistics for overlapping targets
 
    A user reported that restic's statistics and progress information during backup
-   was not correctly calculated when the backup sources (files/dirs to save)
+   was not correctly calculated when the backup targets (files/dirs to save)
    overlap. For example, consider a directory `foo` which contains (among others) a
    file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the
    size of the file `foo/bar` twice, so the completeness percentage as well as the
@@ -3659,7 +2925,7 @@ restic users. The changes are ordered by importance.
  * Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune`
 
    The `copy` and `prune` commands used to traverse the directories of snapshots
-   one by one to find used data. This snapshot traversal is now parallelized which
+   one by one to find used data. This snapshot traversal is now parallized which
    can speed up this step several times.
 
    In addition the `check` command now reports how many snapshots have already been
@@ -3758,10 +3024,11 @@ restic users. The changes are ordered by importance.
 
  * Bugfix #1756: Mark repository files as read-only when using the local backend
 
-   Files stored in a local repository were marked as writable on the filesystem for
+   Files stored in a local repository were marked as writeable on the filesystem
-   non-Windows systems, which did not prevent accidental file modifications outside
+   for non-Windows systems, which did not prevent accidental file modifications
-   of restic. In addition, the local backend did not work with certain filesystems
+   outside of restic. In addition, the local backend did not work with certain
-   and network mounts which do not permit modifications of file permissions.
+   filesystems and network mounts which do not permit modifications of file
+   permissions.
 
    Restic now marks files stored in a local repository as read-only on the
    filesystem on non-Windows systems. The error handling is improved to support
@@ -3855,7 +3122,7 @@ restic users. The changes are ordered by importance.
    was unable to backup those files before. This update enables backing up these
    files.
 
-   This needs to be enabled explicitly using the --use-fs-snapshot option of the
+   This needs to be enabled explicitely using the --use-fs-snapshot option of the
    backup command.
 
    https://github.com/restic/restic/issues/340
@@ -4065,8 +3332,8 @@ restic users. The changes are ordered by importance.
 
  * Bugfix #2668: Don't abort the stats command when data blobs are missing
 
-   Running the stats command in the blobs-per-file mode on a repository with
+   Runing the stats command in the blobs-per-file mode on a repository with missing
-   missing data blobs previously resulted in a crash.
+   data blobs previously resulted in a crash.
 
    https://github.com/restic/restic/pull/2668
 
@@ -4221,7 +3488,7 @@ restic users. The changes are ordered by importance.
 
    NOTE: This new implementation does not guarantee order in which blobs are
    written to the target files and, for example, the last blob of a file can be
-   written to the file before any of the preceding file blobs. It is therefore
+   written to the file before any of the preceeding file blobs. It is therefore
    possible to have gaps in the data written to the target files if restore fails
    or interrupted by the user.
 
@@ -4455,7 +3722,7 @@ restic users. The changes are ordered by importance.
    will be disabled if the --ignore-inode flag was given.
 
    If this change causes problems for you, please open an issue, and we can look in
-   to adding a separate flag to disable just the ctime check.
+   to adding a seperate flag to disable just the ctime check.
 
    https://github.com/restic/restic/issues/2179
    https://github.com/restic/restic/pull/2212
@@ -4829,7 +4096,7 @@ restic users. The changes are ordered by importance.
  * Enhancement #1876: Display reason why forget keeps snapshots
 
    We've added a column to the list of snapshots `forget` keeps which details the
-   reasons to keep a particular snapshot. This makes debugging policies for forget
+   reasons to keep a particuliar snapshot. This makes debugging policies for forget
    much easier. Please remember to always try things out with `--dry-run`!
 
    https://github.com/restic/restic/pull/1876
@@ -5142,7 +4409,7 @@ restic users. The changes are ordered by importance.
  * Enh #1665: Improve cache handling for `restic check`
  * Enh #1709: Improve messages `restic check` prints
  * Enh #1721: Add `cache` command to list cache dirs
- * Enh #1735: Allow keeping a time range of snapshots
+ * Enh #1735: Allow keeping a time range of snaphots
  * Enh #1758: Allow saving OneDrive folders in Windows
  * Enh #1782: Use default AWS credentials chain for S3 backend
 
@@ -5348,7 +4615,7 @@ restic users. The changes are ordered by importance.
    https://github.com/restic/restic/issues/1721
    https://github.com/restic/restic/pull/1749
 
- * Enhancement #1735: Allow keeping a time range of snapshots
+ * Enhancement #1735: Allow keeping a time range of snaphots
 
    We've added the `--keep-within` option to the `forget` command. It instructs
    restic to keep all snapshots within the given duration since the newest
@@ -5450,7 +4717,7 @@ restic users. The changes are ordered by importance.
    already exists.
 
    This is not accurate, the file could have been created between the HTTP request
-   testing for it, and when writing starts, so we've relaxed this requirement,
+   testing for it, and when writing starts, so we've relaxed this requeriment,
    which saves one additional HTTP request per newly added file.
 
    https://github.com/restic/restic/pull/1623
@@ -5470,7 +4737,7 @@ restic users. The changes are ordered by importance.
 
 ## Summary
 
- * Fix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends
+ * Fix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends
  * Fix #1512: Restore directory permissions as the last step
  * Fix #1528: Correctly create missing subdirs in data/
  * Fix #1589: Complete intermediate index upload
@@ -5490,7 +4757,7 @@ restic users. The changes are ordered by importance.
 
 ## Details
 
- * Bugfix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends
+ * Bugfix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends
 
    https://github.com/restic/restic/issues/1506
    https://github.com/restic/restic/pull/1511
@@ -5547,7 +4814,7 @@ restic users. The changes are ordered by importance.
  * Bugfix #1595: Backup: Remove bandwidth display
 
    This commit removes the bandwidth displayed during backup process. It is
-   misleading and seldom correct, because it's neither the "read bandwidth" (only
+   misleading and seldomly correct, because it's neither the "read bandwidth" (only
    for the very first backup) nor the "upload bandwidth". Many users are confused
    about (and rightly so), c.f. #1581, #1033, #1591
 
@@ -5820,7 +5087,7 @@ restic users. The changes are ordered by importance.
    We've added a local cache for metadata so that restic doesn't need to load all
    metadata (snapshots, indexes, ...) from the repo each time it starts. By default
    the cache is active, but there's a new global option `--no-cache` that can be
-   used to disable the cache. By default, the cache a standard cache folder for the
+   used to disable the cache. By deafult, the cache a standard cache folder for the
    OS, which can be overridden with `--cache-dir`. The cache will automatically
    populate, indexes and snapshots are saved as they are loaded. Cache directories
    for repos that haven't been used recently can automatically be removed by restic
@@ -5907,7 +5174,7 @@ restic users. The changes are ordered by importance.
 
  * Enhancement #1319: Make `check` print `no errors found` explicitly
 
-   The `check` command now explicitly prints `No errors were found` when no errors
+   The `check` command now explicetly prints `No errors were found` when no errors
    could be found.
 
    https://github.com/restic/restic/issues/1303

README.md

@@ -10,7 +10,8 @@ For detailed usage and installation instructions check out the [documentation](h
 
 You can ask questions in our [Discourse forum](https://forum.restic.net).
 
-## Quick start
+Quick start
+-----------
 
 Once you've [installed](https://restic.readthedocs.io/en/latest/020_installation.html) restic, start
 off with creating a repository for your backups:
@@ -58,7 +59,7 @@ Therefore, restic supports the following backends for storing backups natively:
 Restic is a program that does backups right and was designed with the
 following principles in mind:
 
--  **Easy**: Doing backups should be a frictionless process, otherwise
+-  **Easy:** Doing backups should be a frictionless process, otherwise
    you might be tempted to skip it. Restic should be easy to configure
    and use, so that, in the event of a data loss, you can just restore
    it. Likewise, restoring data should not be complicated.
@@ -91,17 +92,20 @@ reproduce a byte identical version from the source code for that
 release. Instructions on how to do that are contained in the
 [builder repository](https://github.com/restic/builder).
 
-## News
+News
+----
 
-You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or subscribe to
+You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or by subscribing to
 the [project blog](https://restic.net/blog/).
 
-## License
+License
+-------
 
 Restic is licensed under [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause). You can find the
-complete text in [`LICENSE`](LICENSE).
+complete text in [``LICENSE``](LICENSE).
 
-## Sponsorship
+Sponsorship
+-----------
 
 Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
 Storage are sponsored by [AppsCode](https://appscode.com)!

VERSION

@@ -1 +1 @@
-0.17.0
+0.16.4

changelog/0.10.0_2020-09-19/pull-2195

@@ -10,7 +10,7 @@ https://github.com/restic/restic/issues/2244
 
 NOTE: This new implementation does not guarantee order in which blobs
 are written to the target files and, for example, the last blob of a
-file can be written to the file before any of the preceding file blobs.
+file can be written to the file before any of the preceeding file blobs.
 It is therefore possible to have gaps in the data written to the target
 files if restore fails or interrupted by the user.
 

changelog/0.12.0_2021-02-14/issue-3232

@@ -1,7 +1,7 @@
-Bugfix: Correct statistics for overlapping backup sources
+Bugfix: Correct statistics for overlapping targets
 
 A user reported that restic's statistics and progress information during backup
-was not correctly calculated when the backup sources (files/dirs to save)
+was not correctly calculated when the backup targets (files/dirs to save)
 overlap. For example, consider a directory `foo` which contains (among others)
 a file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the
 size of the file `foo/bar` twice, so the completeness percentage as well as the

changelog/0.16.0_2023-07-31/issue-3941

@@ -1,7 +1,7 @@
 Enhancement: Support `--group-by` for backup parent selection
 
 Previously, the `backup` command by default selected the parent snapshot based
-on the hostname and the backup paths. When the backup path list changed, the
+on the hostname and the backup targets. When the backup path list changed, the
 `backup` command was unable to determine a suitable parent snapshot and had to
 read all files again.
 

changelog/0.16.5_2024-07-01/issue-4873

@@ -1,6 +0,0 @@
-Enhancement: Update dependencies
-
-A few potentially vulnerable dependencies were updated.
-
-https://github.com/restic/restic/issues/4873
-https://github.com/restic/restic/pull/4878

changelog/0.16.5_2024-07-01/pull-4799

@@ -1,5 +0,0 @@
-Enhancement: Add option to force use of Azure CLI credential
-
-A new environment variable `AZURE_FORCE_CLI_CREDENTIAL=true` allows forcing the use of Azure CLI credential, ignoring other credentials like managed identity.
-
-https://github.com/restic/restic/pull/4799

changelog/0.17.0_2024-07-26/issue-1786

@@ -1,20 +0,0 @@
-Enhancement: Support repositories with empty password
-
-Restic previously required a password to create or operate on repositories.
-Using the new option `--insecure-no-password` it is now possible to disable
-this requirement. Restic will not prompt for a password when using this option.
-
-For security reasons, the option must always be specified when operating on
-repositories with an empty password, and specifying `--insecure-no-password`
-while also passing a password to restic via a CLI option or environment
-variable results in an error.
-
-The `init` and `copy` commands add the related `--from-insecure-no-password`
-option, which applies to the source repository. The `key add` and `key passwd`
-commands add the `--new-insecure-no-password` option to add or set an empty
-password.
-
-https://github.com/restic/restic/issues/1786
-https://github.com/restic/restic/issues/4326
-https://github.com/restic/restic/pull/4698
-https://github.com/restic/restic/pull/4808

changelog/0.17.0_2024-07-26/issue-2348

@@ -1,12 +0,0 @@
-Enhancement: Add `--delete` option to `restore` command
-
-The `restore` command now supports a `--delete` option that allows removing
-files and directories from the target directory that do not exist in the
-snapshot. This option also allows files in the snapshot to replace non-empty
-directories having the same name.
-
-To check that only expected files are deleted, add the `--dry-run --verbose=2`
-options.
-
-https://github.com/restic/restic/issues/2348
-https://github.com/restic/restic/pull/4881

changelog/0.17.0_2024-07-26/issue-3600

@@ -1,11 +0,0 @@
-Bugfix: Handle unreadable xattrs in folders above `backup` source
-
-When backup sources are specified using absolute paths, `backup` also includes
-information about the parent folders of the backup sources in the snapshot.
-
-If the extended attributes for some of these folders could not be read due to
-missing permissions, this caused the backup to fail. This has now been fixed.
-
-https://github.com/restic/restic/issues/3600
-https://github.com/restic/restic/pull/4668
-https://forum.restic.net/t/parent-directories-above-the-snapshot-source-path-fatal-error-permission-denied/7216

changelog/0.17.0_2024-07-26/issue-3806

@@ -1,12 +0,0 @@
-Enhancement: Optimize and make `prune` command resumable
-
-Previously, if the `prune` command was interrupted, a later `prune` run would
-start repacking pack files from the start, as `prune` did not update the index
-while repacking.
-
-The `prune` command now supports resuming interrupted prune runs. The update
-of the repository index has also been optimized to use less memory and only
-rewrite parts of the index that have changed.
-
-https://github.com/restic/restic/issues/3806
-https://github.com/restic/restic/pull/4812

changelog/0.17.0_2024-07-26/issue-4048

@@ -1,6 +0,0 @@
-Enhancement: Add support for FUSE-T with `mount` on macOS
-
-The restic `mount` command now supports creating FUSE mounts using FUSE-T on macOS.
-
-https://github.com/restic/restic/issues/4048
-https://github.com/restic/restic/pull/4825

changelog/0.17.0_2024-07-26/issue-4209

@@ -1,7 +0,0 @@
-Bugfix: Fix slow SFTP upload performance
-
-Since restic 0.12.1, the upload speed of the sftp backend to a remote server
-has regressed significantly. This has now been fixed.
-
-https://github.com/restic/restic/issues/4209
-https://github.com/restic/restic/pull/4782

changelog/0.17.0_2024-07-26/issue-4287

@@ -1,12 +0,0 @@
-Enhancement: Support connection to rest-server using unix socket
-
-Restic now supports using a unix socket to connect to a rest-server
-version 0.13.0 or later. This allows running restic as follows:
-
-```
-rest-server --listen unix:/tmp/rest.socket --data /path/to/data &
-restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...]
-```
-
-https://github.com/restic/restic/issues/4287
-https://github.com/restic/restic/pull/4655

changelog/0.17.0_2024-07-26/issue-4437

@@ -1,11 +0,0 @@
-Enhancement: Make `check` command create non-existent cache directory
-
-Previously, if a custom cache directory was specified for the `check` command,
-but the directory did not exist, `check` continued with the cache disabled.
-
-The `check` command now attempts to create the cache directory before
-initializing the cache.
-
-https://github.com/restic/restic/issues/4437
-https://github.com/restic/restic/pull/4805
-https://github.com/restic/restic/pull/4883

changelog/0.17.0_2024-07-26/issue-4547

@@ -1,7 +0,0 @@
-Enhancement: Add `--json` option to `version` command
-
-Restic now supports outputting restic version along with the Go version, OS
-and architecture used to build restic in JSON format using `version --json`.
-
-https://github.com/restic/restic/issues/4547
-https://github.com/restic/restic/pull/4553

changelog/0.17.0_2024-07-26/issue-4549

@@ -1,11 +0,0 @@
-Enhancement: Add `--ncdu` option to `ls` command
-
-NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. It has
-an option to save a directory tree and analyse it later.
-
-The `ls` command now supports outputting snapshot information in the NCDU format
-using the `--ncdu` option. Example usage: `restic ls latest --ncdu | ncdu -f -`
-
-https://github.com/restic/restic/issues/4549
-https://github.com/restic/restic/pull/4550
-https://github.com/restic/restic/pull/4911

changelog/0.17.0_2024-07-26/issue-4568

@@ -1,16 +0,0 @@
-Bugfix: Prevent `forget --keep-tags <invalid>` from deleting all snapshots
-
-Running `forget --keep-tags <invalid>`, where `<invalid>` is a tag that does
-not exist in the repository, would remove all snapshots. This is especially
-problematic if the tag name contains a typo.
-
-The `forget` command now fails with an error if all snapshots in a snapshot
-group would be deleted. This prevents the above example from deleting all
-snapshots.
-
-It is possible to temporarily disable the new check by setting the environment
-variable `RESTIC_FEATURES=safe-forget-keep-tags=false`. Note that this feature
-flag will be removed in the next minor restic version.
-
-https://github.com/restic/restic/pull/4568
-https://github.com/restic/restic/pull/4764

changelog/0.17.0_2024-07-26/issue-4583

@@ -1,13 +0,0 @@
-Enhancement: Ignore `s3.storage-class` archive tiers for metadata
-
-Restic used to store all files on S3 using the specified `s3.storage-class`.
-
-Now, restic will only use non-archive storage tiers for metadata, to avoid
-problems when accessing a repository. To restore any data, it is still
-necessary to manually warm up the required data beforehand.
-
-NOTE: There is no official cold storage support in restic, use this option at
-your own risk.
-
-https://github.com/restic/restic/issues/4583
-https://github.com/restic/restic/pull/4584

changelog/0.17.0_2024-07-26/issue-4601

@@ -1,9 +0,0 @@
-Enhancement: Add support for feature flags
-
-Restic now supports feature flags that can be used to enable and disable
-experimental features. The flags can be set using the environment variable
-`RESTIC_FEATURES`. To get a list of currently supported feature flags, use
-the `features` command.
-
-https://github.com/restic/restic/issues/4601
-https://github.com/restic/restic/pull/4666

changelog/0.17.0_2024-07-26/issue-4602

@@ -1,22 +0,0 @@
-Change: Deprecate legacy index format and `s3legacy` repository layout
-
-Support for the legacy index format used by restic before version 0.2.0 has
-been deprecated and will be removed in the next minor restic version. You can
-use `restic repair index` to update the index to the current format.
-
-It is possible to temporarily reenable support for the legacy index format by
-setting the environment variable `RESTIC_FEATURES=deprecate-legacy-index=false`.
-Note that this feature flag will be removed in the next minor restic version.
-
-Support for the `s3legacy` repository layout used for the S3 backend before
-restic 0.7.0 has been deprecated and will be removed in the next minor restic
-version. You can migrate your S3 repository to the current layout using
-`RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout`.
-
-It is possible to temporarily reenable support for the `s3legacy` layout by
-setting the environment variable `RESTIC_FEATURES=deprecate-s3-legacy-layout=false`.
-Note that this feature flag will be removed in the next minor restic version.
-
-https://github.com/restic/restic/issues/4602
-https://github.com/restic/restic/pull/4724
-https://github.com/restic/restic/pull/4743

changelog/0.17.0_2024-07-26/issue-4627

@@ -1,33 +0,0 @@
-Change: Redesign backend error handling to improve reliability
-
-Restic now downloads pack files in large chunks instead of using a streaming
-download. This prevents failures due to interrupted streams. The `restore`
-command now also retries downloading individual blobs that could not be
-retrieved.
-
-HTTP requests that are stuck for more than two minutes while uploading or
-downloading are now forcibly interrupted. This ensures that stuck requests are
-retried after a short timeout.
-
-Attempts to access a missing or truncated file will no longer be retried. This
-avoids unnecessary retries in those cases. All other backend requests are
-retried for up to 15 minutes. This ensures that temporarily interrupted network
-connections can be tolerated.
-
-If a download yields a corrupt file or blob, then the download will be retried
-once.
-
-Most parts of the new backend error handling can temporarily be disabled by
-setting the environment variable `RESTIC_FEATURES=backend-error-redesign=false`.
-Note that this feature flag will be removed in the next minor restic version.
-
-https://github.com/restic/restic/issues/4627
-https://github.com/restic/restic/issues/4193
-https://github.com/restic/restic/pull/4605
-https://github.com/restic/restic/pull/4792
-https://github.com/restic/restic/issues/4515
-https://github.com/restic/restic/issues/1523
-https://github.com/restic/restic/pull/4520
-https://github.com/restic/restic/pull/4800
-https://github.com/restic/restic/pull/4784
-https://github.com/restic/restic/pull/4844

changelog/0.17.0_2024-07-26/issue-4676

@@ -1,8 +0,0 @@
-Enhancement: Make `key` command's actions separate sub-commands
-
-Each of the `add`, `list`, `remove` and `passwd` actions provided by the `key`
-command is now a separate sub-command and have its own documentation which can
-be invoked using `restic key <add|list|remove|passwd> --help`.
-
-https://github.com/restic/restic/issues/4676
-https://github.com/restic/restic/pull/4685

changelog/0.17.0_2024-07-26/issue-4678

@@ -1,8 +0,0 @@
-Enhancement: Add `--target` option to the `dump` command
-
-Restic `dump` always printed to the standard output. It now supports specifying
-a `--target` file to write its output to.
-
-https://github.com/restic/restic/issues/4678
-https://github.com/restic/restic/pull/4682
-https://github.com/restic/restic/pull/4692

changelog/0.17.0_2024-07-26/issue-4707

@@ -1,14 +0,0 @@
-Change: Disable S3 anonymous authentication by default
-
-When using the S3 backend with anonymous authentication, it continuously
-tried to retrieve new authentication credentials, causing bad performance.
-
-Now, to use anonymous authentication, it is necessary to pass the extended
-option `-o s3.unsafe-anonymous-auth=true` to restic.
-
-It is possible to temporarily revert to the old behavior by setting the
-environment variable `RESTIC_FEATURES=explicit-s3-anonymous-auth=false`. Note
-that this feature flag will be removed in the next minor restic version.
-
-https://github.com/restic/restic/issues/4707
-https://github.com/restic/restic/pull/4908

changelog/0.17.0_2024-07-26/issue-4733

@@ -1,12 +0,0 @@
-Enhancement: Allow specifying `--host` via environment variable
-
-Restic commands that operate on snapshots, such as `restic backup` and
-`restic snapshots`, support the `--host` option to specify the hostname
-for grouping snapshots.
-
-Such commands now also support specifying the hostname via the environment
-variable `RESTIC_HOST`. Note that `--host` still takes precedence over the
-environment variable.
-
-https://github.com/restic/restic/issues/4733
-https://github.com/restic/restic/pull/4734

changelog/0.17.0_2024-07-26/issue-4744

@@ -1,9 +0,0 @@
-Change: Include full key ID in JSON output of `key list`
-
-The JSON output of the `key list` command has changed to include the full key
-ID instead of just a shortened version of the ID, as the latter can be ambiguous
-in some rare cases. To derive the short ID, please truncate the full ID down to
-eight characters.
-
-https://github.com/restic/restic/issues/4744
-https://github.com/restic/restic/pull/4745

changelog/0.17.0_2024-07-26/issue-4760

@@ -1,8 +0,0 @@
-Bugfix: Fix possible error on concurrent cache cleanup
-
-If multiple restic processes concurrently cleaned up no longer existing files
-from the cache, this could cause some of the processes to fail with an `no such
-file or directory` error. This has now been fixed.
-
-https://github.com/restic/restic/issues/4760
-https://github.com/restic/restic/pull/4761

changelog/0.17.0_2024-07-26/issue-4768

@@ -1,8 +0,0 @@
-Enhancement: Allow specifying custom User-Agent for outgoing requests
-
-Restic now supports setting a custom `User-Agent` for outgoing HTTP requests
-using the global option `--http-user-agent` or the `RESTIC_HTTP_USER_AGENT`
-environment variable.
-
-https://github.com/restic/restic/issues/4768
-https://github.com/restic/restic/pull/4810

changelog/0.17.0_2024-07-26/issue-4781

@@ -1,8 +0,0 @@
-Enhancement: Add `restore` options to read include/exclude patterns from files
-
-Restic now supports reading include and exclude patterns from files using the 
-`--include-file`, `--exclude-file`, `--iinclude-file` and `--iexclude-file` 
-options of the `restore` command.
-
-https://github.com/restic/restic/issues/4781
-https://github.com/restic/restic/pull/4811

changelog/0.17.0_2024-07-26/issue-4817

@@ -1,26 +0,0 @@
-Enhancement: Make overwrite behavior of `restore` customizable
-
-The `restore` command now supports an `--overwrite` option to configure whether
-already existing files are overwritten. The overwrite behavior can be configured
-using the following option values:
-
-- `--overwrite always` (default): Always overwrites already existing files.
-  The `restore` command will verify the existing file content and only restore
-  mismatching parts to minimize downloads. Updates the metadata of all files.
-- `--overwrite if-changed`: Like `always`, but speeds up the file content check
-  by assuming that files with matching size and modification time (mtime) are
-  already up to date. In case of a mismatch, the full file content is verified
-  like with `always`. Updates the metadata of all files.
-- `--overwrite if-newer`: Like `always`, but only overwrites existing files
-  when the file in the snapshot has a newer modification time (mtime) than the
-  existing file.
-- `--overwrite never`: Never overwrites existing files.
-
-https://github.com/restic/restic/issues/4817
-https://github.com/restic/restic/issues/200
-https://github.com/restic/restic/issues/407
-https://github.com/restic/restic/issues/2662
-https://github.com/restic/restic/pull/4837
-https://github.com/restic/restic/pull/4838
-https://github.com/restic/restic/pull/4864
-https://github.com/restic/restic/pull/4921

changelog/0.17.0_2024-07-26/issue-4850

@@ -1,8 +0,0 @@
-Bugfix: Handle UTF-16 password files in `key` command correctly
-
-Previously, `key add` and `key passwd` did not properly decode UTF-16
-encoded passwords read from a password file. This has now been fixed
-to correctly match the encoding when opening a repository.
-
-https://github.com/restic/restic/issues/4850
-https://github.com/restic/restic/pull/4851

changelog/0.17.0_2024-07-26/issue-4902

@@ -1,8 +0,0 @@
-Bugfix: Update snapshot summary on `rewrite`
-
-Restic previously did not recalculate the total number of files and bytes
-processed when files were excluded from a snapshot by the `rewrite` command.
-This has now been fixed.
-
-https://github.com/restic/restic/issues/4902
-https://github.com/restic/restic/pull/4905

changelog/0.17.0_2024-07-26/issue-662

@@ -1,11 +0,0 @@
-Enhancement: Optionally skip snapshot creation if nothing changed
-
-The `backup` command always created a snapshot even if nothing in the
-backup set changed compared to the parent snapshot.
-
-Restic now supports the `--skip-if-unchanged` option for the `backup`
-command, which omits creating a snapshot if the new snapshot's content
-would be identical to that of the parent snapshot.
-
-https://github.com/restic/restic/issues/662
-https://github.com/restic/restic/pull/4816

changelog/0.17.0_2024-07-26/issue-693

@@ -1,13 +0,0 @@
-Enhancement: Include snapshot size in `snapshots` output
-
-The `snapshots` command now prints the size for snapshots created using this
-or a future restic version. To achieve this, the `backup` command now stores
-the backup summary statistics in the snapshot.
-
-The text output of the `snapshots` command only shows the snapshot size. The
-other statistics are only included in the JSON output. To inspect these
-statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
-
-https://github.com/restic/restic/issues/693
-https://github.com/restic/restic/pull/4705
-https://github.com/restic/restic/pull/4913

changelog/0.17.0_2024-07-26/issue-828

@@ -1,11 +0,0 @@
-Enhancement: Improve features of the `repair packs` command
-
-The `repair packs` command has been improved to also be able to process
-truncated pack files. The `check` and `check --read-data` command will provide
-instructions on using the command if necessary to repair a repository. See the
-guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html for
-further instructions.
-
-https://github.com/restic/restic/issues/828
-https://github.com/restic/restic/pull/4644
-https://github.com/restic/restic/pull/4882

changelog/0.17.0_2024-07-26/pull-3067

@@ -1,25 +0,0 @@
-Enhancement: Add extended options to configure Windows Shadow Copy Service
-
-Previous, restic always used a 120 seconds timeout and unconditionally created
-VSS snapshots for all volume mount points on disk. This behavior can now be
-fine-tuned by the following new extended options (available only on Windows):
-
-- `-o vss.timeout`: Time that VSS can spend creating snapshot before timing out (default: 120s)
-- `-o vss.exclude-all-mount-points`: Exclude mountpoints from snapshotting on all volumes (default: false)
-- `-o vss.exclude-volumes`: Semicolon separated list of volumes to exclude from snapshotting
-- `-o vss.provider`: VSS provider identifier which will be used for snapshotting
-
-For example, change VSS timeout to five minutes and disable snapshotting of
-mount points on all volumes:
-
-    restic backup --use-fs-snapshot -o vss.timeout=5m -o vss.exclude-all-mount-points=true
-
-Exclude drive `d:`, mount point `c:\mnt` and a specific volume from snapshotting:
-
-    restic backup --use-fs-snapshot -o vss.exclude-volumes="d:\;c:\mnt\;\\?\Volume{e2e0315d-9066-4f97-8343-eb5659b35762}"
-
-Uses 'Microsoft Software Shadow Copy provider 1.0' instead of the default provider:
-
-    restic backup --use-fs-snapshot -o vss.provider={b5946137-7b9f-4925-af80-51abd60b20d5}
-
-https://github.com/restic/restic/pull/3067

changelog/0.17.0_2024-07-26/pull-4006

@@ -1,15 +0,0 @@
-Enhancement: (alpha) Store deviceID only for hardlinks
-
-Set `RESTIC_FEATURES=device-id-for-hardlinks` to enable this alpha feature.
-The feature flag will be removed after repository format version 3 becomes
-available or be replaced with a different solution.
-
-When creating backups from a filesystem snapshot, for example created using
-BTRFS subvolumes, the deviceID of the filesystem changes compared to previous
-snapshots. This prevented restic from deduplicating the directory metadata of
-a snapshot.
-
-When this alpha feature is enabled, the deviceID is only stored for hardlinks,
-which significantly reduces the metadata duplication for most backups.
-
-https://github.com/restic/restic/pull/4006

changelog/0.17.0_2024-07-26/pull-4354

@@ -1,7 +0,0 @@
-Enhancement: Significantly reduce `prune` memory usage
-
-The `prune` command has been optimized to use up to 60% less memory.
-The memory usage should now be roughly similar to creating a backup.
-
-https://github.com/restic/restic/pull/4354
-https://github.com/restic/restic/pull/4812

changelog/0.17.0_2024-07-26/pull-4503

@@ -1,8 +0,0 @@
-Bugfix: Correct hardlink handling in `stats` command
-
-If files on different devices had the same inode ID, the `stats` command
-did not correctly calculate the snapshot size. This has now been fixed.
-
-https://forum.restic.net/t/possible-bug-in-stats/6461/8
-https://github.com/restic/restic/pull/4503
-https://github.com/restic/restic/pull/4006

changelog/0.17.0_2024-07-26/pull-4526

@@ -1,12 +0,0 @@
-Enhancement: Add bitrot detection to `diff` command
-
-The output of the `diff` command now includes the modifier `?` for files to
-indicate bitrot in backed up files. The `?` will appear whenever there is a
-difference in content while the metadata is exactly the same.
-
-Since files with unchanged metadata are normally not read again when creating
-a backup, the detection is only effective when the right-hand side of the diff
-has been created with `backup --force`.
-
-https://github.com/restic/restic/issues/805
-https://github.com/restic/restic/pull/4526

changelog/0.17.0_2024-07-26/pull-4573

@@ -1,6 +0,0 @@
-Enhancement: Support rewriting host and time metadata in snapshots
-
-The `rewrite` command now supports rewriting the host and/or time metadata of
-a snapshot using the new `--new-host` and `--new-time` options.
-
-https://github.com/restic/restic/pull/4573

changelog/0.17.0_2024-07-26/pull-4590

@@ -1,6 +0,0 @@
-Enhancement: Speed up `mount` command's error detection
-
-The `mount` command now checks for the existence of the mountpoint before
-opening the repository, leading to quicker error detection.
-
-https://github.com/restic/restic/pull/4590

changelog/0.17.0_2024-07-26/pull-4611

@@ -1,9 +0,0 @@
-Enhancement: Back up more file metadata on Windows
-
-Previously, restic did not back up all common Windows-specific metadata.
-
-Restic now stores file creation time and file attributes like the hidden,
-read-only and encrypted flags when backing up files and folders on Windows.
-
-https://github.com/restic/restic/pull/4611
-

changelog/0.17.0_2024-07-26/pull-4664

@@ -1,10 +0,0 @@
-Enhancement: Make `ls` use `message_type` field in JSON output
-
-The `ls` command was the only restic command that used the `struct_type` field
-in its JSON output format to specify the message type.
-
-The JSON output of the `ls` command now also includes the `message_type` field,
-which is consistent with other commands. The `struct_type` field is still
-included, but now deprecated.
-
-https://github.com/restic/restic/pull/4664

changelog/0.17.0_2024-07-26/pull-4703

@@ -1,11 +0,0 @@
-Bugfix: Shutdown cleanly when receiving SIGTERM
-
-Previously, when restic received the SIGTERM signal it would terminate
-immediately, skipping cleanup and potentially causing issues like stale locks
-being left behind. This primarily effected containerized restic invocations
-that use SIGTERM, but could also be triggered via a simple `killall restic`.
-
-This has now been fixed, such that restic shuts down cleanly when receiving
-the SIGTERM signal.
-
-https://github.com/restic/restic/pull/4703

changelog/0.17.0_2024-07-26/pull-4708

@@ -1,13 +0,0 @@
-Enhancement: Back up and restore SecurityDescriptors on Windows
-
-Restic now backs up and restores SecurityDescriptors for files and folders on
-Windows which includes owner, group, discretionary access control list (DACL)
-and system access control list (SACL).
-
-This requires the user to be a member of backup operators or the application
-must be run as admin. If that is not the case, only the current user's owner,
-group and DACL will be backed up, and during restore only the DACL of the
-backed up file will be restored, with the current user's owner and group
-being set on the restored file.
-
-https://github.com/restic/restic/pull/4708

changelog/0.17.0_2024-07-26/pull-4709

@@ -1,10 +0,0 @@
-Bugfix: Correct `--no-lock` handling of `ls` and `tag` commands
-
-The `ls` command never locked the repository. This has now been fixed, with the
-old behavior still being supported using `ls --no-lock`. The latter invocation
-also works with older restic versions.
-
-The `tag` command erroneously accepted the `--no-lock` command. This command
-now always requires an exclusive lock.
-
-https://github.com/restic/restic/pull/4709

changelog/0.17.0_2024-07-26/pull-4737

@@ -1,6 +0,0 @@
-Enhancement: Include snapshot ID in `reason` field of `forget` JSON output
-
-The JSON output of the `forget` command now includes `id` and `short_id` of
-snapshots in the `reason` field.
-
-https://github.com/restic/restic/pull/4737

changelog/0.17.0_2024-07-26/pull-4764

@@ -1,10 +0,0 @@
-Enhancement: Support forgetting all snapshots
-
-The `forget` command now supports the `--unsafe-allow-remove-all` option, which
-removes all snapshots in the repository.
-
-This option must always be combined with a snapshot filter (by host, path or
-tag). For example, the command `forget --tag example --unsafe-allow-remove-all`
-removes all snapshots with the tag "example".
-
-https://github.com/restic/restic/pull/4764

changelog/0.17.0_2024-07-26/pull-4796

@@ -1,8 +0,0 @@
-Enhancement: Improve `dump` performance for large files
-
-The `dump` command now retrieves the data chunks for a file in
-parallel. This improves the download performance by up to as many
-times as the configured number of parallel backend connections.
-
-https://github.com/restic/restic/issues/3406
-https://github.com/restic/restic/pull/4796

changelog/0.17.0_2024-07-26/pull-4807

@@ -1,6 +0,0 @@
-Enhancement: Support Extended Attributes on Windows NTFS
-
-Restic now backs up and restores Extended Attributes for files
-and folders on Windows NTFS.
-
-https://github.com/restic/restic/pull/4807

changelog/0.17.0_2024-07-26/pull-4839

@@ -1,7 +0,0 @@
-Enhancement: Add dry-run support to `restore` command
-
-The `restore` command now supports the `--dry-run` option to perform
-a dry run. Pass the `--verbose=2` option to see which files would
-remain unchanged, and which would be updated or freshly restored.
-
-https://github.com/restic/restic/pull/4839

changelog/0.17.0_2024-07-26/pull-4884

@@ -1,11 +0,0 @@
-Change: Return exit code 10 and 11 for non-existing and locked repository
-
-If a repository does not exist or cannot be locked, restic previously always
-returned exit code 1. This made it difficult to distinguish these cases from
-other errors.
-
-Restic now returns exit code 10 if the repository does not exist, and exit code
-11 if the repository could be not locked due to a conflicting lock.
-
-https://github.com/restic/restic/issues/956
-https://github.com/restic/restic/pull/4884

changelog/unreleased/issue-4251

@@ -1,4 +1,4 @@
-Enhancement: Support reading backup from a command's standard output
+Enhancement: Support reading backup from a commands's standard output
 
 The `backup` command now supports the `--stdin-from-command` option. When using
 this option, the arguments to `backup` are interpreted as a command instead of

changelog/unreleased/issue-4472

@@ -1,11 +1,11 @@
-Enhancement: Support AWS Assume Role for S3 backend
+Enhancement: Allow AWS Assume Role to be used for S3 backend
 
 Previously only credentials discovered via the Minio discovery methods
 were used to authenticate.
 
 However, there are many circumstances where the discovered credentials have
 lower permissions and need to assume a specific role. This is now possible
-using the following new environment variables:
+using the following new environment variables.
 
 - RESTIC_AWS_ASSUME_ROLE_ARN
 - RESTIC_AWS_ASSUME_ROLE_SESSION_NAME

changelog/unreleased/issue-4515

@@ -0,0 +1,8 @@
+Change: Don't retry to load files that don't exist
+
+Restic used to always retry to load files. It now only retries to load
+files if they exist.
+
+https://github.com/restic/restic/issues/4515
+https://github.com/restic/restic/issues/1523
+https://github.com/restic/restic/pull/4520

changelog/unreleased/issue-4540

@@ -1,7 +1,6 @@
 Change: Require at least ARMv6 for ARM binaries
 
-The official release binaries of restic now require
+The official release binaries of restic now require at least ARMv6 support for ARM platforms.
-at least ARMv6 support for ARM platforms.
 
 https://github.com/restic/restic/issues/4540
 https://github.com/restic/restic/pull/4542

changelog/unreleased/issue-4547

@@ -0,0 +1,7 @@
+Enhancement: Add support for `--json` option to `version` command
+
+Restic now supports outputting restic version and used go version, OS and
+architecture via JSON when using the version command.
+
+https://github.com/restic/restic/issues/4547
+https://github.com/restic/restic/pull/4553

changelog/unreleased/issue-4549

@@ -0,0 +1,11 @@
+Enhancement: Add `--ncdu` option to `ls` command
+
+NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories.
+It has an option to save a directory tree and analyse it later.
+The `ls` command now supports the `--ncdu` option which outputs information
+about a snapshot in the NCDU format.
+
+You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
+
+https://github.com/restic/restic/issues/4549
+https://github.com/restic/restic/pull/4550

changelog/unreleased/issue-4583

@@ -0,0 +1,12 @@
+Enhancement: Ignore s3.storage-class for metadata if archive tier is specified
+
+There is no official cold storage support in restic, use this option at your
+own risk.
+
+Restic always stored all files on s3 using the specified `s3.storage-class`.
+Now, restic will store metadata using a non-archive storage tier to avoid
+problems when accessing a repository. To restore any data, it is still
+necessary to manually warm up the required data beforehand.
+
+https://github.com/restic/restic/issues/4583
+https://github.com/restic/restic/pull/4584

changelog/unreleased/issue-4656

@@ -1,6 +1,6 @@
-Bugfix: Properly report ID of newly added keys
+Bugfix: Properly report the ID of newly added keys
 
-`restic key add` now reports the ID of the newly added key. This simplifies
+`restic key add` now reports the ID of a newly added key. This simplifies
 selecting a specific key using the `--key-hint key` option.
 
 https://github.com/restic/restic/issues/4656

changelog/unreleased/issue-4676

@@ -0,0 +1,8 @@
+Enhancement: Move key add, list, remove and passwd as separate sub-commands
+
+Restic now provides usage documentation for the `key` command. Each sub-command;
+`add`, `list`, `remove` and `passwd` now have their own sub-command documentation
+which can be invoked using `restic key <add|list|remove|passwd> --help`.
+
+https://github.com/restic/restic/issues/4676
+https://github.com/restic/restic/pull/4685

changelog/unreleased/issue-4678

@@ -0,0 +1,8 @@
+Enhancement: Add --target flag to the dump command
+
+Restic `dump` always printed to the standard output. It now permits to select a
+`--target` file to write the output to.
+
+https://github.com/restic/restic/issues/4678
+https://github.com/restic/restic/pull/4682
+https://github.com/restic/restic/pull/4692

changelog/unreleased/pull-4503

@@ -0,0 +1,7 @@
+Bugfix: Correct hardlink handling in `stats` command
+
+If files on different devices had the same inode id, then the `stats` command
+did not correctly calculate the snapshot size. This has been fixed.
+
+https://github.com/restic/restic/pull/4503
+https://forum.restic.net/t/possible-bug-in-stats/6461/8

changelog/unreleased/pull-4526

@@ -0,0 +1,11 @@
+Enhancement: Add bitrot detection to `diff` command
+
+The output of the `diff` command now includes the modifier `?` for files
+to indicate bitrot in backed up files. It will appear whenever there is a
+difference in content while the metadata is exactly the same. Since files with
+unchanged metadata are normally not read again when creating a backup, the
+detection is only effective if the right-hand side of the diff has been created
+with "backup --force".
+
+https://github.com/restic/restic/issues/805
+https://github.com/restic/restic/pull/4526

changelog/unreleased/pull-4573

@@ -0,0 +1,5 @@
+Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command
+
+`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot.
+
+https://github.com/restic/restic/pull/4573

changelog/unreleased/pull-4590

@@ -0,0 +1,7 @@
+Enhancement: `mount` tests mountpoint existence before opening the repository
+
+The restic `mount` command now checks for the existence of the
+mountpoint before opening the repository, leading to quicker error
+detection.
+
+https://github.com/restic/restic/pull/4590

changelog/unreleased/pull-4615

@@ -1,6 +1,6 @@
-Bugfix: Make `find` not sometimes ignore directories
+Bugfix: `find` ignored directories in some cases
 
 In some cases, the `find` command ignored empty or moved directories. This has
-now been fixed.
+been fixed.
 
 https://github.com/restic/restic/pull/4615

changelog/unreleased/pull-4644

@@ -0,0 +1,10 @@
+Enhancement: Improve `repair packs` command
+
+The `repair packs` command has been improved to also be able to process
+truncated pack files. The `check --read-data` command will provide instructions
+on using the command if necessary to repair a repository. See the guide at
+https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further
+instructions.
+
+https://github.com/restic/restic/pull/4644
+https://github.com/restic/restic/pull/4655

cmd/restic/cleanup.go

@@ -1,41 +1,89 @@
 package main
 
 import (
-	"context"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/restic/restic/internal/debug"
 )
 
-func createGlobalContext() context.Context {
+var cleanupHandlers struct {
-	ctx, cancel := context.WithCancel(context.Background())
+	sync.Mutex
-
+	list []func(code int) (int, error)
-	ch := make(chan os.Signal, 1)
+	done bool
-	go cleanupHandler(ch, cancel)
+	ch   chan os.Signal
-	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
-
-	return ctx
 }
 
-// cleanupHandler handles the SIGINT and SIGTERM signals.
+func init() {
-func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
+	cleanupHandlers.ch = make(chan os.Signal, 1)
-	s := <-c
+	go CleanupHandler(cleanupHandlers.ch)
-	debug.Log("signal %v received, cleaning up", s)
+	signal.Notify(cleanupHandlers.ch, syscall.SIGINT)
-	Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
+}
 
-	if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
+// AddCleanupHandler adds the function f to the list of cleanup handlers so
-		_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
+// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
-		_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
+// is received.
-		_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
+func AddCleanupHandler(f func(code int) (int, error)) {
+	cleanupHandlers.Lock()
+	defer cleanupHandlers.Unlock()
+
+	// reset the done flag for integration tests
+	cleanupHandlers.done = false
+
+	cleanupHandlers.list = append(cleanupHandlers.list, f)
+}
+
+// RunCleanupHandlers runs all registered cleanup handlers
+func RunCleanupHandlers(code int) int {
+	cleanupHandlers.Lock()
+	defer cleanupHandlers.Unlock()
+
+	if cleanupHandlers.done {
+		return code
 	}
+	cleanupHandlers.done = true
 
-	cancel()
+	for _, f := range cleanupHandlers.list {
+		var err error
+		code, err = f(code)
+		if err != nil {
+			Warnf("error in cleanup handler: %v\n", err)
+		}
+	}
+	cleanupHandlers.list = nil
+	return code
 }
 
-// Exit terminates the process with the given exit code.
+// CleanupHandler handles the SIGINT signals.
+func CleanupHandler(c <-chan os.Signal) {
+	for s := range c {
+		debug.Log("signal %v received, cleaning up", s)
+		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
+
+		if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
+			_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
+			_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
+			_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
+		}
+
+		code := 0
+
+		if s == syscall.SIGINT {
+			code = 130
+		} else {
+			code = 1
+		}
+
+		Exit(code)
+	}
+}
+
+// Exit runs the cleanup handlers and then terminates the process with the
+// given exit code.
 func Exit(code int) {
+	code = RunCleanupHandlers(code)
 	debug.Log("exiting with status code %d", code)
 	os.Exit(code)
 }

cmd/restic/cmd_backup.go

@@ -41,8 +41,6 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was a fatal error (no snapshot created).
 Exit status is 3 if some source data could not be read (incomplete snapshot created).
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	PreRun: func(_ *cobra.Command, _ []string) {
 		if backupOptions.Host == "" {
@@ -89,7 +87,6 @@ type BackupOptions struct {
 	DryRun            bool
 	ReadConcurrency   uint
 	NoScan            bool
-	SkipIfUnchanged   bool
 }
 
 var backupOptions BackupOptions
@@ -104,7 +101,7 @@ func init() {
 	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
 	backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
 	f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
-	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
+	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 
 	initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
 
@@ -117,7 +114,7 @@ func init() {
 	f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
 	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
 	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
-	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
 	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
@@ -136,16 +133,10 @@ func init() {
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
-	f.BoolVar(&backupOptions.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
 
 	// parse read concurrency from env, on error the default value will be used
 	readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
 	backupOptions.ReadConcurrency = uint(readConcurrency)
-
-	// parse host from env, if not exists or empty the default value will be used
-	if host := os.Getenv("RESTIC_HOST"); host != "" {
-		backupOptions.Host = host
-	}
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -162,7 +153,7 @@ func filterExisting(items []string) (result []string, err error) {
 	}
 
 	if len(result) == 0 {
-		return nil, errors.Fatal("all source directories/files do not exist")
+		return nil, errors.Fatal("all target directories/files do not exist")
 	}
 
 	return
@@ -261,7 +252,7 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
 
 // Check returns an error when an invalid combination of options was set.
 func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
-	if gopts.password == "" && !gopts.InsecureNoPassword {
+	if gopts.password == "" {
 		if opts.Stdin {
 			return errors.Fatal("cannot read both password and data from stdin")
 		}
@@ -407,7 +398,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 	// and have the ability to use both files-from and args at the same time.
 	targets = append(targets, args...)
 	if len(targets) == 0 && !opts.Stdin {
-		return nil, errors.Fatal("nothing to backup, please specify source files/dirs")
+		return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
 	}
 
 	targets, err = filterExisting(targets)
@@ -449,16 +440,7 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o
 }
 
 func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
-	var vsscfg fs.VSSConfig
+	err := opts.Check(gopts, args)
-	var err error
-
-	if runtime.GOOS == "windows" {
-		if vsscfg, err = fs.ParseVSSConfig(gopts.extended); err != nil {
-			return err
-		}
-	}
-
-	err = opts.Check(gopts, args)
 	if err != nil {
 		return err
 	}
@@ -469,7 +451,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	timeStamp := time.Now()
-	backupStart := timeStamp
 	if opts.TimeStamp != "" {
 		timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
 		if err != nil {
@@ -481,11 +462,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		Verbosef("open repository\n")
 	}
 
-	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
 	var progressPrinter backup.ProgressPrinter
 	if gopts.JSON {
@@ -497,6 +477,22 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 	defer progressReporter.Done()
 
+	if opts.DryRun {
+		repo.SetDryRun()
+	}
+
+	if !gopts.JSON {
+		progressPrinter.V("lock repository")
+	}
+	if !opts.DryRun {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
 	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
 	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
 	if err != nil {
@@ -560,8 +556,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 			return err
 		}
 
-		errorHandler := func(item string, err error) {
+		errorHandler := func(item string, err error) error {
-			_ = progressReporter.Error(item, err)
+			return progressReporter.Error(item, err)
 		}
 
 		messageHandler := func(msg string, args ...interface{}) {
@@ -570,7 +566,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 			}
 		}
 
-		localVss := fs.NewLocalVss(errorHandler, messageHandler, vsscfg)
+		localVss := fs.NewLocalVss(errorHandler, messageHandler)
 		defer localVss.DeleteSnapshots()
 		targetFS = localVss
 	}
@@ -642,20 +638,18 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	snapshotOpts := archiver.SnapshotOptions{
-		Excludes:        opts.Excludes,
+		Excludes:       opts.Excludes,
-		Tags:            opts.Tags.Flatten(),
+		Tags:           opts.Tags.Flatten(),
-		BackupStart:     backupStart,
+		Time:           timeStamp,
-		Time:            timeStamp,
+		Hostname:       opts.Host,
-		Hostname:        opts.Host,
+		ParentSnapshot: parentSnapshot,
-		ParentSnapshot:  parentSnapshot,
+		ProgramVersion: "restic " + version,
-		ProgramVersion:  "restic " + version,
-		SkipIfUnchanged: opts.SkipIfUnchanged,
 	}
 
 	if !gopts.JSON {
 		progressPrinter.V("start backup on %v", targets)
 	}
-	_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
+	_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
 
 	// cleanly shutdown all running goroutines
 	cancel()
@@ -669,7 +663,10 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	// Report finished execution
-	progressReporter.Finish(id, summary, opts.DryRun)
+	progressReporter.Finish(id, opts.DryRun)
+	if !gopts.JSON && !opts.DryRun {
+		progressPrinter.P("snapshot %s saved\n", id.Str())
+	}
 	if !success {
 		return ErrInvalidSourceData
 	}

cmd/restic/cmd_backup_integration_test.go

@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -249,18 +250,29 @@ func TestBackupTreeLoadError(t *testing.T) {
 	opts := BackupOptions{}
 	// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
 	testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
-	treePacks := listTreePacks(env.gopts, t)
+
+	r, err := OpenRepository(context.TODO(), env.gopts)
+	rtest.OK(t, err)
+	rtest.OK(t, r.LoadIndex(context.TODO(), nil))
+	treePacks := restic.NewIDSet()
+	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
+		}
+	})
 
 	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
 
 	// delete the subdirectory pack first
-	removePacks(env.gopts, t, treePacks)
+	for id := range treePacks {
+		rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
+	}
 	testRunRebuildIndex(t, env.gopts)
 	// now the repo is missing the tree blob in the index; check should report this
 	testRunCheckMustFail(t, env.gopts)
 	// second backup should report an error but "heal" this situation
-	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
 	testRunCheck(t, env.gopts)
 
@@ -394,7 +406,6 @@ func TestIncrementalBackup(t *testing.T) {
 	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
 }
 
-// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupTags(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -430,7 +441,6 @@ func TestBackupTags(t *testing.T) {
 		"expected parent to be %v, got %v", parent.ID, newest.Parent)
 }
 
-// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupProgramVersion(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -627,32 +637,3 @@ func TestStdinFromCommandFailNoOutputAndExitCode(t *testing.T) {
 
 	testRunCheck(t, env.gopts)
 }
-
-func TestBackupEmptyPassword(t *testing.T) {
-	// basic sanity test that empty passwords work
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
-	env.gopts.password = ""
-	env.gopts.InsecureNoPassword = true
-
-	testSetupBackupData(t, env)
-	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
-	testListSnapshots(t, env.gopts, 1)
-	testRunCheck(t, env.gopts)
-}
-
-func TestBackupSkipIfUnchanged(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
-	testSetupBackupData(t, env)
-	opts := BackupOptions{SkipIfUnchanged: true}
-
-	for i := 0; i < 3; i++ {
-		testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
-		testListSnapshots(t, env.gopts, 1)
-	}
-
-	testRunCheck(t, env.gopts)
-}

cmd/restic/cmd_cache.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/restic/restic/internal/backend/cache"
+	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/ui"
@@ -25,8 +25,7 @@ The "cache" command allows listing and cleaning local cache directories.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(_ *cobra.Command, args []string) error {

cmd/restic/cmd_cat.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -21,10 +22,7 @@ The "cat" command is used to print internal objects to stdout.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -66,11 +64,19 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	tpe := args[0]
 
@@ -148,9 +154,9 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return nil
 
 	case "pack":
-		buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
+		h := backend.Handle{Type: restic.PackFile, Name: id.String()}
-		// allow returning broken pack files
+		buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
-		if buf == nil {
+		if err != nil {
 			return err
 		}
 
@@ -170,7 +176,8 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		}
 
 		for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
-			if _, ok := repo.LookupBlobSize(t, id); !ok {
+			bh := restic.BlobHandle{ID: id, Type: t}
+			if !repo.Index().Has(bh) {
 				continue
 			}
 

cmd/restic/cmd_check.go

@@ -11,15 +11,12 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/restic/restic/internal/backend/cache"
+	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
-	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
-	"github.com/restic/restic/internal/ui/progress"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 var cmdCheck = &cobra.Command{
@@ -35,16 +32,11 @@ repository and not use a local cache.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
+		return runCheck(cmd.Context(), checkOptions, globalOptions, args)
-		defer cancel()
-		return runCheck(cmd.Context(), checkOptions, globalOptions, args, term)
 	},
 	PreRunE: func(_ *cobra.Command, _ []string) error {
 		return checkFlags(checkOptions)
@@ -162,7 +154,7 @@ func parsePercentage(s string) (float64, error) {
 //   - if the user explicitly requested --no-cache, we don't use any cache
 //   - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
 //   - by default, we use a cache in a temporary directory that is deleted after the check
-func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress.Printer) (cleanup func()) {
+func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
 	cleanup = func() {}
 	if opts.WithCache {
 		// use the default cache, no setup needed
@@ -179,54 +171,53 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
 		cachedir = cache.EnvDir()
 	}
 
-	if cachedir != "" {
+	// use a cache in a temporary directory
-		// use a cache in a temporary directory
-		err := os.MkdirAll(cachedir, 0755)
-		if err != nil {
-			Warnf("unable to create cache directory %s, disabling cache: %v\n", cachedir, err)
-			gopts.NoCache = true
-			return cleanup
-		}
-	}
 	tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-")
 	if err != nil {
 		// if an error occurs, don't use any cache
-		printer.E("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
+		Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
 		gopts.NoCache = true
 		return cleanup
 	}
 
 	gopts.CacheDir = tempdir
-	printer.P("using temporary cache in %v\n", tempdir)
+	Verbosef("using temporary cache in %v\n", tempdir)
 
 	cleanup = func() {
 		err := fs.RemoveAll(tempdir)
 		if err != nil {
-			printer.E("error removing temporary cache directory: %v\n", err)
+			Warnf("error removing temporary cache directory: %v\n", err)
 		}
 	}
 
 	return cleanup
 }
 
-func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error {
+func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
 		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
 	}
 
-	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+	cleanup := prepareCheckCache(opts, &gopts)
+	AddCleanupHandler(func(code int) (int, error) {
+		cleanup()
+		return code, nil
+	})
 
-	cleanup := prepareCheckCache(opts, &gopts, printer)
+	repo, err := OpenRepository(ctx, gopts)
-	defer cleanup()
-
-	if !gopts.NoLock {
-		printer.P("create exclusive lock for repository\n")
-	}
-	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock {
+		Verbosef("create exclusive lock for repository\n")
+		var lock *restic.Lock
+		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	chkr := checker.New(repo, opts.CheckUnused)
 	err = chkr.LoadSnapshots(ctx)
@@ -234,99 +225,71 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		return err
 	}
 
-	printer.P("load indexes\n")
+	Verbosef("load indexes\n")
-	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
+	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	hints, errs := chkr.LoadIndex(ctx, bar)
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
 
 	errorsFound := false
 	suggestIndexRebuild := false
-	suggestLegacyIndexRebuild := false
 	mixedFound := false
 	for _, hint := range hints {
 		switch hint.(type) {
-		case *checker.ErrDuplicatePacks:
+		case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
-			term.Print(hint.Error())
+			Printf("%v\n", hint)
 			suggestIndexRebuild = true
-		case *checker.ErrOldIndexFormat:
-			printer.E("error: %v\n", hint)
-			suggestLegacyIndexRebuild = true
-			errorsFound = true
 		case *checker.ErrMixedPack:
-			term.Print(hint.Error())
+			Printf("%v\n", hint)
 			mixedFound = true
 		default:
-			printer.E("error: %v\n", hint)
+			Warnf("error: %v\n", hint)
 			errorsFound = true
 		}
 	}
 
 	if suggestIndexRebuild {
-		term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
+		Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
-	}
-	if suggestLegacyIndexRebuild {
-		printer.E("error: Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
 	}
 	if mixedFound {
-		term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
+		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
 	}
 
 	if len(errs) > 0 {
 		for _, err := range errs {
-			printer.E("error: %v\n", err)
+			Warnf("error: %v\n", err)
 		}
-
+		return errors.Fatal("LoadIndex returned errors")
-		printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n")
-		return errors.Fatal("repository contains errors")
 	}
 
 	orphanedPacks := 0
 	errChan := make(chan error)
-	salvagePacks := restic.NewIDSet()
 
-	printer.P("check all packs\n")
+	Verbosef("check all packs\n")
 	go chkr.Packs(ctx, errChan)
 
 	for err := range errChan {
-		var packErr *checker.PackError
+		if checker.IsOrphanedPack(err) {
-		if errors.As(err, &packErr) {
+			orphanedPacks++
-			if packErr.Orphaned {
+			Verbosef("%v\n", err)
-				orphanedPacks++
-				printer.V("%v\n", err)
-			} else {
-				if packErr.Truncated {
-					salvagePacks.Insert(packErr.ID)
-				}
-				errorsFound = true
-				printer.E("%v\n", err)
-			}
 		} else if err == checker.ErrLegacyLayout {
-			errorsFound = true
+			Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
-			printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n")
 		} else {
 			errorsFound = true
-			printer.E("%v\n", err)
+			Warnf("%v\n", err)
 		}
 	}
 
-	if orphanedPacks > 0 && !errorsFound {
+	if orphanedPacks > 0 {
-		// hide notice if repository is damaged
+		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
-		printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
-	}
-	if ctx.Err() != nil {
-		return ctx.Err()
 	}
 
-	printer.P("check snapshots, trees and blobs\n")
+	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
 	var wg sync.WaitGroup
 
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		bar := newTerminalProgressMax(!gopts.Quiet, 0, "snapshots", term)
+		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
 		chkr.Structure(ctx, bar, errChan)
 	}()
@@ -334,12 +297,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	for err := range errChan {
 		errorsFound = true
 		if e, ok := err.(*checker.TreeError); ok {
-			printer.E("error for tree %v:\n", e.ID.Str())
+			var clean string
+			if stdoutCanUpdateStatus() {
+				clean = clearLine(0)
+			}
+			Warnf(clean+"error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
-				printer.E("  %v\n", treeErr)
+				Warnf("  %v\n", treeErr)
 			}
 		} else {
-			printer.E("error: %v\n", err)
+			Warnf("error: %v\n", err)
 		}
 	}
 
@@ -347,17 +314,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	// Must happen after `errChan` is read from in the above loop to avoid
 	// deadlocking in the case of errors.
 	wg.Wait()
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
 
 	if opts.CheckUnused {
-		unused, err := chkr.UnusedBlobs(ctx)
+		for _, id := range chkr.UnusedBlobs(ctx) {
-		if err != nil {
+			Verbosef("unused blob %v\n", id)
-			return err
-		}
-		for _, id := range unused {
-			printer.P("unused blob %v\n", id)
 			errorsFound = true
 		}
 	}
@@ -365,24 +325,36 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	doReadData := func(packs map[restic.ID]int64) {
 		packCount := uint64(len(packs))
 
-		p := newTerminalProgressMax(!gopts.Quiet, packCount, "packs", term)
+		p := newProgressMax(!gopts.Quiet, packCount, "packs")
 		errChan := make(chan error)
 
 		go chkr.ReadPacks(ctx, packs, p, errChan)
 
+		var salvagePacks restic.IDs
+
 		for err := range errChan {
 			errorsFound = true
-			printer.E("%v\n", err)
+			Warnf("%v\n", err)
-			if err, ok := err.(*repository.ErrPackData); ok {
+			if err, ok := err.(*checker.ErrPackData); ok {
-				salvagePacks.Insert(err.PackID)
+				salvagePacks = append(salvagePacks, err.PackID)
 			}
 		}
 		p.Done()
+
+		if len(salvagePacks) > 0 {
+			Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
+			var strIDs []string
+			for _, id := range salvagePacks {
+				strIDs = append(strIDs, id.String())
+			}
+			Warnf("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
+			Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
+		}
 	}
 
 	switch {
 	case opts.ReadData:
-		printer.P("read all data\n")
+		Verbosef("read all data\n")
 		doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
 	case opts.ReadDataSubset != "":
 		var packs map[restic.ID]int64
@@ -392,12 +364,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 			totalBuckets := dataSubset[1]
 			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
 			packCount := uint64(len(packs))
-			printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
+			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
 		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
 			percentage, err := parsePercentage(opts.ReadDataSubset)
 			if err == nil {
 				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
-				printer.P("read %.1f%% of data packs\n", percentage)
+				Verbosef("read %.1f%% of data packs\n", percentage)
 			}
 		} else {
 			repoSize := int64(0)
@@ -413,7 +385,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 				subsetSize = repoSize
 			}
 			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
-			printer.P("read %d bytes of data packs\n", subsetSize)
+			Verbosef("read %d bytes of data packs\n", subsetSize)
 		}
 		if packs == nil {
 			return errors.Fatal("internal error: failed to select packs to check")
@@ -421,27 +393,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		doReadData(packs)
 	}
 
-	if len(salvagePacks) > 0 {
-		printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
-		var strIDs []string
-		for id := range salvagePacks {
-			strIDs = append(strIDs, id.String())
-		}
-		printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
-		printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
-	}
-
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
-
 	if errorsFound {
-		if len(salvagePacks) == 0 {
-			printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n")
-		}
 		return errors.Fatal("repository contains errors")
 	}
-	printer.P("no errors were found\n")
+
+	Verbosef("no errors were found\n")
 
 	return nil
 }

cmd/restic/cmd_check_integration_test.go

@@ -1,12 +1,10 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"testing"
 
 	rtest "github.com/restic/restic/internal/test"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func testRunCheck(t testing.TB, gopts GlobalOptions) {
@@ -25,14 +23,12 @@ func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
 }
 
 func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
-	buf := bytes.NewBuffer(nil)
+	buf, err := withCaptureStdout(func() error {
-	gopts.stdout = buf
-	err := withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
 		opts := CheckOptions{
 			ReadData:    true,
 			CheckUnused: checkUnused,
 		}
-		return runCheck(context.TODO(), opts, gopts, nil, term)
+		return runCheck(context.TODO(), opts, gopts, nil)
 	})
 	return buf.String(), err
 }

cmd/restic/cmd_check_test.go

@@ -1,17 +1,12 @@
 package main
 
 import (
-	"io/fs"
 	"math"
-	"os"
 	"reflect"
-	"strings"
 	"testing"
 
-	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
-	"github.com/restic/restic/internal/ui/progress"
 )
 
 func TestParsePercentage(t *testing.T) {
@@ -168,79 +163,3 @@ func TestSelectNoRandomPacksByFileSize(t *testing.T) {
 	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
 }
-
-func checkIfFileWithSimilarNameExists(files []fs.DirEntry, fileName string) bool {
-	found := false
-	for _, file := range files {
-		if file.IsDir() {
-			dirName := file.Name()
-			if strings.Contains(dirName, fileName) {
-				found = true
-			}
-		}
-	}
-	return found
-}
-
-func TestPrepareCheckCache(t *testing.T) {
-	// Create a temporary directory for the cache
-	tmpDirBase := t.TempDir()
-
-	testCases := []struct {
-		opts           CheckOptions
-		withValidCache bool
-	}{
-		{CheckOptions{WithCache: true}, true},   // Shouldn't create temp directory
-		{CheckOptions{WithCache: false}, true},  // Should create temp directory
-		{CheckOptions{WithCache: false}, false}, // Should create cache directory first, then temp directory
-	}
-
-	for _, testCase := range testCases {
-		t.Run("", func(t *testing.T) {
-			if !testCase.withValidCache {
-				// remove tmpDirBase to simulate non-existing cache directory
-				err := os.Remove(tmpDirBase)
-				rtest.OK(t, err)
-			}
-			gopts := GlobalOptions{CacheDir: tmpDirBase}
-			cleanup := prepareCheckCache(testCase.opts, &gopts, &progress.NoopPrinter{})
-			files, err := os.ReadDir(tmpDirBase)
-			rtest.OK(t, err)
-
-			if !testCase.opts.WithCache {
-				// If using a temporary cache directory, the cache directory should exist
-				// listing all directories inside tmpDirBase (cacheDir)
-				// one directory should be tmpDir created by prepareCheckCache with 'restic-check-cache-' in path
-				found := checkIfFileWithSimilarNameExists(files, "restic-check-cache-")
-				if !found {
-					t.Errorf("Expected temporary directory to exist, but it does not")
-				}
-			} else {
-				// If not using the cache, the temp directory should not exist
-				rtest.Assert(t, len(files) == 0, "expected cache directory not to exist, but it does: %v", files)
-			}
-
-			// Call the cleanup function to remove the temporary cache directory
-			cleanup()
-
-			// Verify that the cache directory has been removed
-			files, err = os.ReadDir(tmpDirBase)
-			rtest.OK(t, err)
-			rtest.Assert(t, len(files) == 0, "Expected cache directory to be removed, but it still exists: %v", files)
-		})
-	}
-}
-
-func TestPrepareDefaultCheckCache(t *testing.T) {
-	gopts := GlobalOptions{CacheDir: ""}
-	cleanup := prepareCheckCache(CheckOptions{}, &gopts, &progress.NoopPrinter{})
-	_, err := os.ReadDir(gopts.CacheDir)
-	rtest.OK(t, err)
-
-	// Call the cleanup function to remove the temporary cache directory
-	cleanup()
-
-	// Verify that the cache directory has been removed
-	_, err = os.ReadDir(gopts.CacheDir)
-	rtest.Assert(t, errors.Is(err, os.ErrNotExist), "Expected cache directory to be removed, but it still exists")
-}

cmd/restic/cmd_copy.go

@@ -30,14 +30,6 @@ This means that copied files, which existed in both the source and destination
 repository, /may occupy up to twice their space/ in the destination repository.
 This can be mitigated by the "--copy-chunker-params" option when initializing a
 new destination repository using the "init" command.
-
-EXIT STATUS
-===========
-
-Exit status is 0 if the command was successful.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCopy(cmd.Context(), copyOptions, globalOptions, args)
@@ -61,7 +53,7 @@ func init() {
 }
 
 func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
-	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
+	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
 	if err != nil {
 		return err
 	}
@@ -70,17 +62,30 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		gopts, secondaryGopts = secondaryGopts, gopts
 	}
 
-	ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	srcRepo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
-	ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false)
+	dstRepo, err := OpenRepository(ctx, secondaryGopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		var srcLock *restic.Lock
+		srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(srcLock)
+		if err != nil {
+			return err
+		}
+	}
+
+	dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(dstLock)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
 	srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
 	if err != nil {
@@ -111,9 +116,6 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		// also consider identical snapshot copies
 		dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
 	}
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
 
 	// remember already processed trees across all snapshots
 	visitedTrees := restic.NewIDSet()
@@ -158,7 +160,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		}
 		Verbosef("snapshot %s saved\n", newID.Str())
 	}
-	return ctx.Err()
+	return nil
 }
 
 func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
@@ -195,7 +197,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	packList := restic.NewIDSet()
 
 	enqueue := func(h restic.BlobHandle) {
-		pb := srcRepo.LookupBlob(h.Type, h.ID)
+		pb := srcRepo.Index().Lookup(h)
 		copyBlobs.Insert(h)
 		for _, p := range pb {
 			packList.Insert(p.PackID)
@@ -210,7 +212,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 
 			// Do we already have this tree blob?
 			treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
-			if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok {
+			if !dstRepo.Index().Has(treeHandle) {
 				// copy raw tree bytes to avoid problems if the serialization changes
 				enqueue(treeHandle)
 			}
@@ -220,7 +222,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 				// Copy the blobs for this file.
 				for _, blobID := range entry.Content {
 					h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
-					if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
+					if !dstRepo.Index().Has(h) {
 						enqueue(h)
 					}
 				}

cmd/restic/cmd_copy_integration_test.go

@@ -13,12 +13,10 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
 	gopts := srcGopts
 	gopts.Repo = dstGopts.Repo
 	gopts.password = dstGopts.password
-	gopts.InsecureNoPassword = dstGopts.InsecureNoPassword
 	copyOpts := CopyOptions{
 		secondaryRepoOptions: secondaryRepoOptions{
-			Repo:               srcGopts.Repo,
+			Repo:     srcGopts.Repo,
-			password:           srcGopts.password,
+			password: srcGopts.password,
-			InsecureNoPassword: srcGopts.InsecureNoPassword,
 		},
 	}
 
@@ -136,22 +134,3 @@ func TestCopyUnstableJSON(t *testing.T) {
 	testRunCheck(t, env2.gopts)
 	testListSnapshots(t, env2.gopts, 1)
 }
-
-func TestCopyToEmptyPassword(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-	env2, cleanup2 := withTestEnvironment(t)
-	defer cleanup2()
-	env2.gopts.password = ""
-	env2.gopts.InsecureNoPassword = true
-
-	testSetupBackupData(t, env)
-	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, BackupOptions{}, env.gopts)
-
-	testRunInit(t, env2.gopts)
-	testRunCopy(t, env.gopts, env2.gopts)
-
-	testListSnapshots(t, env.gopts, 1)
-	testListSnapshots(t, env2.gopts, 1)
-	testRunCheck(t, env2.gopts)
-}

cmd/restic/cmd_debug.go

@@ -20,11 +20,12 @@ import (
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/repository/index"
-	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
 )
 
@@ -43,10 +44,7 @@ is used for debugging purposes only.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -155,11 +153,19 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 		return errors.Fatal("type not specified")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	tpe := args[0]
 
@@ -318,11 +324,10 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
 	if err != nil {
 		panic(err)
 	}
-
+	be := repo.Backend()
-	pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
+	h := backend.Handle{
-	// allow processing broken pack files
+		Name: packID.String(),
-	if pack == nil {
+		Type: restic.PackFile,
-		return err
 	}
 
 	wg, ctx := errgroup.WithContext(ctx)
@@ -334,11 +339,19 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
 	wg.Go(func() error {
 		for _, blob := range list {
 			Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
-			if int(blob.Offset+blob.Length) > len(pack) {
+			buf := make([]byte, blob.Length)
-				Warnf("skipping truncated blob\n")
+			err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
+				n, err := io.ReadFull(rd, buf)
+				if err != nil {
+					return fmt.Errorf("read error after %d bytes: %v", n, err)
+				}
+				return nil
+			})
+			if err != nil {
+				Warnf("error read: %v\n", err)
 				continue
 			}
-			buf := pack[blob.Offset : blob.Offset+blob.Length]
+
 			key := repo.Key()
 
 			nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
@@ -429,15 +442,10 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 }
 
 func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
-	if opts.ExtractPack && gopts.NoLock {
+	repo, err := OpenRepository(ctx, gopts)
-		return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive")
-	}
-
-	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
 	ids := make([]restic.ID, 0)
 	for _, name := range args {
@@ -456,6 +464,15 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamine
 		return errors.Fatal("no pack files to examine")
 	}
 
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
 	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
@@ -477,12 +494,20 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamine
 func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error {
 	Printf("examine %v\n", id)
 
-	buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
+	h := backend.Handle{
-	// also process damaged pack files
+		Type: restic.PackFile,
-	if buf == nil {
+		Name: id.String(),
+	}
+	fi, err := repo.Backend().Stat(ctx, h)
+	if err != nil {
+		return err
+	}
+	Printf("  file size is %v\n", fi.Size)
+
+	buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
+	if err != nil {
 		return err
 	}
-	Printf("  file size is %v\n", len(buf))
 	gotID := restic.Hash(buf)
 	if !id.Equal(gotID) {
 		Printf("  wanted hash %v, got %v\n", id, gotID)
@@ -495,13 +520,13 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo
 
 	blobsLoaded := false
 	// examine all data the indexes have for the pack file
-	for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
+	for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) {
 		blobs := b.Blobs
 		if len(blobs) == 0 {
 			continue
 		}
 
-		checkPackSize(blobs, len(buf))
+		checkPackSize(blobs, fi.Size)
 
 		err = loadBlobs(ctx, opts, repo, id, blobs)
 		if err != nil {
@@ -514,11 +539,11 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo
 	Printf("  ========================================\n")
 	Printf("  inspect the pack itself\n")
 
-	blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
+	blobs, _, err := repo.ListPack(ctx, id, fi.Size)
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}
-	checkPackSize(blobs, len(buf))
+	checkPackSize(blobs, fi.Size)
 
 	if !blobsLoaded {
 		return loadBlobs(ctx, opts, repo, id, blobs)
@@ -526,7 +551,7 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo
 	return nil
 }
 
-func checkPackSize(blobs []restic.Blob, fileSize int) {
+func checkPackSize(blobs []restic.Blob, fileSize int64) {
 	// track current size and offset
 	var size, offset uint64
 

cmd/restic/cmd_diff.go

@@ -33,16 +33,13 @@ Metadata comparison will likely not work if a backup was created using the
 '--ignore-inode' or '--ignore-ctime' option.
 
 To only compare files in specific subfolders, you can use the
-"snapshotID:subfolder" syntax, where "subfolder" is a path within the
+"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
 snapshot.
 
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -159,7 +156,7 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
 			stats.TreeBlobs++
 		}
 
-		size, found := repo.LookupBlobSize(h.Type, h.ID)
+		size, found := repo.LookupBlobSize(h.ID, h.Type)
 		if !found {
 			Warnf("unable to find blob size for %v\n", h)
 			continue
@@ -347,11 +344,19 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		return errors.Fatalf("specify two snapshot IDs")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	// cache snapshots listing
 	be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)

cmd/restic/cmd_dump.go

@@ -28,16 +28,13 @@ The special snapshotID "latest" can be used to use the latest snapshot in the
 repository.
 
 To include the folder content at the root of the archive, you can use the
-"snapshotID:subfolder" syntax, where "subfolder" is a path within the
+"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
 snapshot.
 
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -134,11 +131,19 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 
 	splittedPath := splitPath(path.Clean(pathToPrint))
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,

cmd/restic/cmd_features.go

@@ -1,59 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/feature"
-	"github.com/restic/restic/internal/ui/table"
-
-	"github.com/spf13/cobra"
-)
-
-var featuresCmd = &cobra.Command{
-	Use:   "features",
-	Short: "Print list of feature flags",
-	Long: `
-The "features" command prints a list of supported feature flags.
-
-To pass feature flags to restic, set the RESTIC_FEATURES environment variable
-to "featureA=true,featureB=false". Specifying an unknown feature flag is an error.
-
-A feature can either be in alpha, beta, stable or deprecated state.
-An _alpha_ feature is disabled by default and may change in arbitrary ways between restic versions or be removed.
-A _beta_ feature is enabled by default, but still can change in minor ways or be removed.
-A _stable_ feature is always enabled and cannot be disabled. The flag will be removed in a future restic version.
-A _deprecated_ feature is always disabled and cannot be enabled. The flag will be removed in a future restic version.
-
-EXIT STATUS
-===========
-
-Exit status is 0 if the command was successful.
-Exit status is 1 if there was any error.
-`,
-	Hidden:            true,
-	DisableAutoGenTag: true,
-	RunE: func(_ *cobra.Command, args []string) error {
-		if len(args) != 0 {
-			return errors.Fatal("the feature command expects no arguments")
-		}
-
-		fmt.Printf("All Feature Flags:\n")
-		flags := feature.Flag.List()
-
-		tab := table.New()
-		tab.AddColumn("Name", "{{ .Name }}")
-		tab.AddColumn("Type", "{{ .Type }}")
-		tab.AddColumn("Default", "{{ .Default }}")
-		tab.AddColumn("Description", "{{ .Description }}")
-
-		for _, flag := range flags {
-			tab.AddRow(flag)
-		}
-		return tab.Write(globalOptions.stdout)
-	},
-}
-
-func init() {
-	cmdRoot.AddCommand(featuresCmd)
-}

cmd/restic/cmd_find.go

@@ -33,10 +33,7 @@ restic find --pack 025c1d06
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -129,7 +126,6 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
 		// Make the following attributes disappear
 		Name               byte `json:"name,omitempty"`
 		ExtendedAttributes byte `json:"extended_attributes,omitempty"`
-		GenericAttributes  byte `json:"generic_attributes,omitempty"`
 		Device             byte `json:"device,omitempty"`
 		Content            byte `json:"content,omitempty"`
 		Subtree            byte `json:"subtree,omitempty"`
@@ -442,10 +438,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 
 	if err != errAllPacksFound {
 		// try to resolve unknown pack ids from the index
-		packIDs, err = f.indexPacksToBlobs(ctx, packIDs)
+		packIDs = f.indexPacksToBlobs(ctx, packIDs)
-		if err != nil {
-			return err
-		}
 	}
 
 	if len(packIDs) > 0 {
@@ -462,13 +455,13 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 	return nil
 }
 
-func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) (map[string]struct{}, error) {
+func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
 	wctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	// remember which packs were found in the index
 	indexPackIDs := make(map[string]struct{})
-	err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) {
+	f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
 		idStr := pb.PackID.String()
 		// keep entry in packIDs as Each() returns individual index entries
 		matchingID := false
@@ -487,9 +480,6 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 			indexPackIDs[idStr] = struct{}{}
 		}
 	})
-	if err != nil {
-		return nil, err
-	}
 
 	for id := range indexPackIDs {
 		delete(packIDs, id)
@@ -502,17 +492,19 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 		}
 		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
 	}
-	return packIDs, nil
+	return packIDs
 }
 
 func (f *Finder) findObjectPack(id string, t restic.BlobType) {
+	idx := f.repo.Index()
+
 	rid, err := restic.ParseID(id)
 	if err != nil {
 		Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err)
 		return
 	}
 
-	blobs := f.repo.LookupBlob(t, rid)
+	blobs := idx.Lookup(restic.BlobHandle{ID: rid, Type: t})
 	if len(blobs) == 0 {
 		Printf("Object %s not found in the index\n", rid.Str())
 		return
@@ -570,11 +562,19 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		return errors.Fatal("cannot have several ID types")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
@@ -615,9 +615,6 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
 		filteredSnapshots = append(filteredSnapshots, sn)
 	}
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
 
 	sort.Slice(filteredSnapshots, func(i, j int) bool {
 		return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)

cmd/restic/cmd_forget.go

@@ -3,14 +3,11 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"strconv"
 
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
 )
 
@@ -21,9 +18,6 @@ var cmdForget = &cobra.Command{
 The "forget" command removes snapshots according to a policy. All snapshots are
 first divided into groups according to "--group-by", and after that the policy
 specified by the "--keep-*" options is applied to each group individually.
-If there are not enough snapshots to keep one for each duration related
-"--keep-{within-,}*" option, the oldest snapshot in the group is kept
-additionally.
 
 Please note that this command really only deletes the snapshot object in the
 repository, which is a reference to data stored there. In order to remove the
@@ -35,16 +29,11 @@ security considerations.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
+		return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, args)
-		defer cancel()
-		return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args)
 	},
 }
 
@@ -99,8 +88,6 @@ type ForgetOptions struct {
 	WithinYearly  restic.Duration
 	KeepTags      restic.TagLists
 
-	UnsafeAllowRemoveAll bool
-
 	restic.SnapshotFilter
 	Compact bool
 
@@ -130,7 +117,6 @@ func init() {
 	f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
-	f.BoolVar(&forgetOptions.UnsafeAllowRemoveAll, "unsafe-allow-remove-all", false, "allow deleting all snapshots of a snapshot group")
 
 	initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false)
 	f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
@@ -166,7 +152,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
 	return nil
 }
 
-func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
+func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, args []string) error {
 	err := verifyForgetOptions(&opts)
 	if err != nil {
 		return err
@@ -177,21 +163,23 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 		return err
 	}
 
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
 	if gopts.NoLock && !opts.DryRun {
 		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
 	}
 
-	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
+	if !opts.DryRun || !gopts.NoLock {
-	if err != nil {
+		var lock *restic.Lock
-		return err
+		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
 	}
-	defer unlock()
-
-	verbosity := gopts.verbosity
-	if gopts.JSON {
-		verbosity = 0
-	}
-	printer := newTerminalProgressPrinter(verbosity, term)
 
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
@@ -199,9 +187,6 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
 
 	var jsonGroups []*ForgetGroup
 
@@ -232,87 +217,72 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 			Tags:          opts.KeepTags,
 		}
 
-		if policy.Empty() {
+		if policy.Empty() && len(args) == 0 {
-			if opts.UnsafeAllowRemoveAll {
+			if !gopts.JSON {
-				if opts.SnapshotFilter.Empty() {
+				Verbosef("no policy was specified, no snapshots will be removed\n")
-					return errors.Fatal("--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified")
-				}
-				// UnsafeAllowRemoveAll together with snapshot filter is fine
-			} else {
-				return errors.Fatal("no policy was specified, no snapshots will be removed")
 			}
 		}
 
-		printer.P("Applying Policy: %v\n", policy)
+		if !policy.Empty() {
+			if !gopts.JSON {
+				Verbosef("Applying Policy: %v\n", policy)
+			}
 
-		for k, snapshotGroup := range snapshotGroups {
+			for k, snapshotGroup := range snapshotGroups {
-			if gopts.Verbose >= 1 && !gopts.JSON {
+				if gopts.Verbose >= 1 && !gopts.JSON {
-				err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
+					err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
-				if err != nil {
+					if err != nil {
+						return err
+					}
+				}
+
+				var key restic.SnapshotGroupKey
+				if json.Unmarshal([]byte(k), &key) != nil {
 					return err
 				}
-			}
 
-			var key restic.SnapshotGroupKey
+				var fg ForgetGroup
-			if json.Unmarshal([]byte(k), &key) != nil {
+				fg.Tags = key.Tags
-				return err
+				fg.Host = key.Hostname
-			}
+				fg.Paths = key.Paths
 
-			var fg ForgetGroup
+				keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
-			fg.Tags = key.Tags
-			fg.Host = key.Hostname
-			fg.Paths = key.Paths
 
-			keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
+				if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
+					Printf("keep %d snapshots:\n", len(keep))
+					PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
+					Printf("\n")
+				}
+				addJSONSnapshots(&fg.Keep, keep)
 
-			if feature.Flag.Enabled(feature.SafeForgetKeepTags) && !policy.Empty() && len(keep) == 0 {
+				if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
-				return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String())
+					Printf("remove %d snapshots:\n", len(remove))
-			}
+					PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
-			if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
+					Printf("\n")
-				printer.P("keep %d snapshots:\n", len(keep))
+				}
-				PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
+				addJSONSnapshots(&fg.Remove, remove)
-				printer.P("\n")
-			}
-			fg.Keep = asJSONSnapshots(keep)
 
-			if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
+				fg.Reasons = reasons
-				printer.P("remove %d snapshots:\n", len(remove))
-				PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
-				printer.P("\n")
-			}
-			fg.Remove = asJSONSnapshots(remove)
 
-			fg.Reasons = asJSONKeeps(reasons)
+				jsonGroups = append(jsonGroups, &fg)
 
-			jsonGroups = append(jsonGroups, &fg)
+				for _, sn := range remove {
-
+					removeSnIDs.Insert(*sn.ID())
-			for _, sn := range remove {
+				}
-				removeSnIDs.Insert(*sn.ID())
 			}
 		}
 	}
 
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
-
 	if len(removeSnIDs) > 0 {
 		if !opts.DryRun {
-			bar := printer.NewCounter("files deleted")
+			err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
-			err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error {
-				if err != nil {
-					printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
-				} else {
-					printer.VV("removed %v/%v\n", restic.SnapshotFile, id)
-				}
-				return nil
-			}, bar)
-			bar.Done()
 			if err != nil {
 				return err
 			}
 		} else {
-			printer.P("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
+			if !gopts.JSON {
+				Printf("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
+			}
 		}
 	}
 
@@ -324,13 +294,15 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 	}
 
 	if len(removeSnIDs) > 0 && opts.Prune {
-		if opts.DryRun {
+		if !gopts.JSON {
-			printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
+			if opts.DryRun {
-		} else {
+				Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
-			printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+			} else {
+				Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+			}
 		}
 		pruneOptions.DryRun = opts.DryRun
-		return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
+		return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
 	}
 
 	return nil
@@ -338,47 +310,23 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 
 // ForgetGroup helps to print what is forgotten in JSON.
 type ForgetGroup struct {
-	Tags    []string     `json:"tags"`
+	Tags    []string            `json:"tags"`
-	Host    string       `json:"host"`
+	Host    string              `json:"host"`
-	Paths   []string     `json:"paths"`
+	Paths   []string            `json:"paths"`
-	Keep    []Snapshot   `json:"keep"`
+	Keep    []Snapshot          `json:"keep"`
-	Remove  []Snapshot   `json:"remove"`
+	Remove  []Snapshot          `json:"remove"`
-	Reasons []KeepReason `json:"reasons"`
+	Reasons []restic.KeepReason `json:"reasons"`
 }
 
-func asJSONSnapshots(list restic.Snapshots) []Snapshot {
+func addJSONSnapshots(js *[]Snapshot, list restic.Snapshots) {
-	var resultList []Snapshot
 	for _, sn := range list {
 		k := Snapshot{
 			Snapshot: sn,
 			ID:       sn.ID(),
 			ShortID:  sn.ID().Str(),
 		}
-		resultList = append(resultList, k)
+		*js = append(*js, k)
 	}
-	return resultList
-}
-
-// KeepReason helps to print KeepReasons as JSON with Snapshots with their ID included.
-type KeepReason struct {
-	Snapshot Snapshot `json:"snapshot"`
-	Matches  []string `json:"matches"`
-}
-
-func asJSONKeeps(list []restic.KeepReason) []KeepReason {
-	var resultList []KeepReason
-	for _, keep := range list {
-		k := KeepReason{
-			Snapshot: Snapshot{
-				Snapshot: keep.Snapshot,
-				ID:       keep.Snapshot.ID(),
-				ShortID:  keep.Snapshot.ID().Str(),
-			},
-			Matches: keep.Matches,
-		}
-		resultList = append(resultList, k)
-	}
-	return resultList
 }
 
 func printJSONForget(stdout io.Writer, forgets []*ForgetGroup) error {

cmd/restic/cmd_forget_integration_test.go

@@ -2,65 +2,15 @@ package main
 
 import (
 	"context"
-	"path/filepath"
-	"strings"
 	"testing"
 
-	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-func testRunForgetMayFail(gopts GlobalOptions, opts ForgetOptions, args ...string) error {
+func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
+	opts := ForgetOptions{}
 	pruneOpts := PruneOptions{
 		MaxUnused: "5%",
 	}
-	return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+	rtest.OK(t, runForget(context.TODO(), opts, pruneOpts, gopts, args))
-		return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
-	})
-}
-
-func testRunForget(t testing.TB, gopts GlobalOptions, opts ForgetOptions, args ...string) {
-	rtest.OK(t, testRunForgetMayFail(gopts, opts, args...))
-}
-
-func TestRunForgetSafetyNet(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
-	testSetupBackupData(t, env)
-
-	opts := BackupOptions{
-		Host: "example",
-	}
-	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
-	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
-	testListSnapshots(t, env.gopts, 2)
-
-	// --keep-tags invalid
-	err := testRunForgetMayFail(env.gopts, ForgetOptions{
-		KeepTags: restic.TagLists{restic.TagList{"invalid"}},
-		GroupBy:  restic.SnapshotGroupByOptions{Host: true, Path: true},
-	})
-	rtest.Assert(t, strings.Contains(err.Error(), `refusing to delete last snapshot of snapshot group "host example, path`), "wrong error message got %v", err)
-
-	// disallow `forget --unsafe-allow-remove-all`
-	err = testRunForgetMayFail(env.gopts, ForgetOptions{
-		UnsafeAllowRemoveAll: true,
-	})
-	rtest.Assert(t, strings.Contains(err.Error(), `--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified`), "wrong error message got %v", err)
-
-	// disallow `forget` without options
-	err = testRunForgetMayFail(env.gopts, ForgetOptions{})
-	rtest.Assert(t, strings.Contains(err.Error(), `no policy was specified, no snapshots will be removed`), "wrong error message got %v", err)
-
-	// `forget --host example --unsafe-allow-remove-all` should work
-	testRunForget(t, env.gopts, ForgetOptions{
-		UnsafeAllowRemoveAll: true,
-		GroupBy:              restic.SnapshotGroupByOptions{Host: true, Path: true},
-		SnapshotFilter: restic.SnapshotFilter{
-			Hosts: []string{opts.Host},
-		},
-	})
-	testListSnapshots(t, env.gopts, 0)
 }

cmd/restic/cmd_generate.go

@@ -18,8 +18,7 @@ and the auto-completion files for bash, fish and zsh).
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(_ *cobra.Command, args []string) error {

cmd/restic/cmd_init.go

@@ -23,8 +23,7 @@ The "init" command initializes a new repository.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -81,7 +80,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	gopts.password, err = ReadPasswordTwice(ctx, gopts,
+	gopts.password, err = ReadPasswordTwice(gopts,
 		"enter password for new repository: ",
 		"enter password again: ")
 	if err != nil {
@@ -132,7 +131,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 
 func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
 	if opts.CopyChunkerParameters {
-		otherGopts, _, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "secondary")
+		otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
 		if err != nil {
 			return nil, err
 		}

cmd/restic/cmd_key_add.go

@@ -3,11 +3,12 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
+	"strings"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 )
 
 var cmdKeyAdd = &cobra.Command{
@@ -19,36 +20,29 @@ The "add" sub-command creates a new key and validates the key. Returns the new k
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command is successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 	`,
 	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args)
+	},
 }
 
 type KeyAddOptions struct {
-	NewPasswordFile    string
+	NewPasswordFile string
-	InsecureNoPassword bool
+	Username        string
-	Username           string
+	Hostname        string
-	Hostname           string
 }
 
-func (opts *KeyAddOptions) Add(flags *pflag.FlagSet) {
+var keyAddOpts KeyAddOptions
-	flags.StringVarP(&opts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
-	flags.BoolVar(&opts.InsecureNoPassword, "new-insecure-no-password", false, "add an empty password for the repository (insecure)")
-	flags.StringVarP(&opts.Username, "user", "", "", "the username for new key")
-	flags.StringVarP(&opts.Hostname, "host", "", "", "the hostname for new key")
-}
 
 func init() {
 	cmdKey.AddCommand(cmdKeyAdd)
 
-	var keyAddOpts KeyAddOptions
+	flags := cmdKeyAdd.Flags()
-	keyAddOpts.Add(cmdKeyAdd.Flags())
+	flags.StringVarP(&keyAddOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
-	cmdKeyAdd.RunE = func(cmd *cobra.Command, args []string) error {
+	flags.StringVarP(&keyAddOpts.Username, "user", "", "", "the username for new key")
-		return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args)
+	flags.StringVarP(&keyAddOpts.Hostname, "host", "", "", "the hostname for new key")
-	}
 }
 
 func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error {
@@ -56,17 +50,22 @@ func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, arg
 		return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags")
 	}
 
-	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
 	return addKey(ctx, repo, gopts, opts)
 }
 
 func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error {
-	pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword)
+	pw, err := getNewPassword(gopts, opts.NewPasswordFile)
 	if err != nil {
 		return err
 	}
@@ -89,41 +88,33 @@ func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOption
 // testKeyNewPassword is used to set a new password during integration testing.
 var testKeyNewPassword string
 
-func getNewPassword(ctx context.Context, gopts GlobalOptions, newPasswordFile string, insecureNoPassword bool) (string, error) {
+func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) {
 	if testKeyNewPassword != "" {
 		return testKeyNewPassword, nil
 	}
 
-	if insecureNoPassword {
-		if newPasswordFile != "" {
-			return "", fmt.Errorf("only either --new-password-file or --new-insecure-no-password may be specified")
-		}
-		return "", nil
-	}
-
 	if newPasswordFile != "" {
-		password, err := loadPasswordFromFile(newPasswordFile)
+		return loadPasswordFromFile(newPasswordFile)
-		if err != nil {
-			return "", err
-		}
-		if password == "" {
-			return "", fmt.Errorf("an empty password is not allowed by default. Pass the flag `--new-insecure-no-password` to restic to disable this check")
-		}
-		return password, nil
 	}
 
 	// Since we already have an open repository, temporary remove the password
 	// to prompt the user for the passwd.
 	newopts := gopts
 	newopts.password = ""
-	// empty passwords are already handled above
-	newopts.InsecureNoPassword = false
 
-	return ReadPasswordTwice(ctx, newopts,
+	return ReadPasswordTwice(newopts,
 		"enter new password: ",
 		"enter password again: ")
 }
 
+func loadPasswordFromFile(pwdFile string) (string, error) {
+	s, err := os.ReadFile(pwdFile)
+	if os.IsNotExist(err) {
+		return "", errors.Fatalf("%s does not exist", pwdFile)
+	}
+	return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
+}
+
 func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
 	// Verify new key to make sure it really works. A broken key can render the
 	// whole repository inaccessible

cmd/restic/cmd_key_integration_test.go

@@ -3,8 +3,6 @@ package main
 import (
 	"bufio"
 	"context"
-	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
@@ -111,43 +109,6 @@ func TestKeyAddRemove(t *testing.T) {
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
 }
 
-func TestKeyAddInvalid(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-	testRunInit(t, env.gopts)
-
-	err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
-		NewPasswordFile:    "some-file",
-		InsecureNoPassword: true,
-	}, []string{})
-	rtest.Assert(t, strings.Contains(err.Error(), "only either"), "unexpected error message, got %q", err)
-
-	pwfile := filepath.Join(t.TempDir(), "pwfile")
-	rtest.OK(t, os.WriteFile(pwfile, []byte{}, 0o666))
-
-	err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
-		NewPasswordFile: pwfile,
-	}, []string{})
-	rtest.Assert(t, strings.Contains(err.Error(), "an empty password is not allowed by default"), "unexpected error message, got %q", err)
-}
-
-func TestKeyAddEmpty(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	// must list keys more than once
-	env.gopts.backendTestHook = nil
-	defer cleanup()
-	testRunInit(t, env.gopts)
-
-	rtest.OK(t, runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
-		InsecureNoPassword: true,
-	}, []string{}))
-
-	env.gopts.password = ""
-	env.gopts.InsecureNoPassword = true
-
-	testRunCheck(t, env.gopts)
-}
-
 type emptySaveBackend struct {
 	backend.Backend
 }

cmd/restic/cmd_key_list.go

@@ -23,10 +23,7 @@ used to access the repository.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command is successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 	`,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -43,11 +40,19 @@ func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	return listKeys(ctx, repo, gopts)
 }
@@ -56,7 +61,6 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 	type keyInfo struct {
 		Current  bool   `json:"current"`
 		ID       string `json:"id"`
-		ShortID  string `json:"-"`
 		UserName string `json:"userName"`
 		HostName string `json:"hostName"`
 		Created  string `json:"created"`
@@ -74,8 +78,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 
 		key := keyInfo{
 			Current:  id == s.KeyID(),
-			ID:       id.String(),
+			ID:       id.Str(),
-			ShortID:  id.Str(),
 			UserName: k.Username,
 			HostName: k.Hostname,
 			Created:  k.Created.Local().Format(TimeFormat),
@@ -96,7 +99,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 	}
 
 	tab := table.New()
-	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ShortID }}")
+	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
 	tab.AddColumn("User", "{{ .UserName }}")
 	tab.AddColumn("Host", "{{ .HostName }}")
 	tab.AddColumn("Created", "{{ .Created }}")

cmd/restic/cmd_key_passwd.go

@@ -19,26 +19,27 @@ Returns the new key ID.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command is successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 	`,
 	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args)
+	},
 }
 
 type KeyPasswdOptions struct {
 	KeyAddOptions
 }
 
+var keyPasswdOpts KeyPasswdOptions
+
 func init() {
 	cmdKey.AddCommand(cmdKeyPasswd)
 
-	var keyPasswdOpts KeyPasswdOptions
+	flags := cmdKeyPasswd.Flags()
-	keyPasswdOpts.KeyAddOptions.Add(cmdKeyPasswd.Flags())
+	flags.StringVarP(&keyPasswdOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
-	cmdKeyPasswd.RunE = func(cmd *cobra.Command, args []string) error {
+	flags.StringVarP(&keyPasswdOpts.Username, "user", "", "", "the username for new key")
-		return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args)
+	flags.StringVarP(&keyPasswdOpts.Hostname, "host", "", "", "the hostname for new key")
-	}
 }
 
 func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error {
@@ -46,17 +47,22 @@ func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOption
 		return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags")
 	}
 
-	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	repo, err := OpenRepository(ctx, gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
 	return changePassword(ctx, repo, gopts, opts)
 }
 
 func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error {
-	pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword)
+	pw, err := getNewPassword(gopts, opts.NewPasswordFile)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_key_remove.go

@@ -20,10 +20,7 @@ removing the current key being used to access the repository.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command is successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 	`,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -40,13 +37,20 @@ func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error
 		return fmt.Errorf("key remove expects one argument as the key id")
 	}
 
-	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
-	return deleteKey(ctx, repo, args[0])
+	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	idPrefix := args[0]
+
+	return deleteKey(ctx, repo, idPrefix)
 }
 
 func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error {

cmd/restic/cmd_list.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository/index"
+	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
@@ -19,10 +19,7 @@ The "list" command allows listing objects in the repository based on type.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -39,11 +36,19 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("type not specified")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock || args[0] == "locks")
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
+
+	if !gopts.NoLock && args[0] != "locks" {
+		var lock *restic.Lock
+		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
 
 	var t restic.FileType
 	switch args[0] {
@@ -62,9 +67,10 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
 			if err != nil {
 				return err
 			}
-			return idx.Each(ctx, func(blobs restic.PackedBlob) {
+			idx.Each(ctx, func(blobs restic.PackedBlob) {
 				Printf("%v %v\n", blobs.Type, blobs.ID)
 			})
+			return nil
 		})
 	default:
 		return errors.Fatal("invalid type")

cmd/restic/cmd_ls.go

@@ -39,10 +39,7 @@ a path separator); paths use the forward slash '/' as separator.
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
-Exit status is 1 if there was any error.
-Exit status is 10 if the repository does not exist.
-Exit status is 11 if the repository is already locked.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -74,7 +71,7 @@ func init() {
 
 type lsPrinter interface {
 	Snapshot(sn *restic.Snapshot)
-	Node(path string, node *restic.Node, isPrefixDirectory bool)
+	Node(path string, node *restic.Node)
 	LeaveDir(path string)
 	Close()
 }
@@ -86,18 +83,16 @@ type jsonLsPrinter struct {
 func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
 	type lsSnapshot struct {
 		*restic.Snapshot
-		ID          *restic.ID `json:"id"`
+		ID         *restic.ID `json:"id"`
-		ShortID     string     `json:"short_id"`
+		ShortID    string     `json:"short_id"`
-		MessageType string     `json:"message_type"` // "snapshot"
+		StructType string     `json:"struct_type"` // "snapshot"
-		StructType  string     `json:"struct_type"`  // "snapshot", deprecated
 	}
 
 	err := p.enc.Encode(lsSnapshot{
-		Snapshot:    sn,
+		Snapshot:   sn,
-		ID:          sn.ID(),
+		ID:         sn.ID(),
-		ShortID:     sn.ID().Str(),
+		ShortID:    sn.ID().Str(),
-		MessageType: "snapshot",
+		StructType: "snapshot",
-		StructType:  "snapshot",
 	})
 	if err != nil {
 		Warnf("JSON encode failed: %v\n", err)
@@ -105,10 +100,7 @@ func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
 }
 
 // Print node in our custom JSON format, followed by a newline.
-func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
+func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
-	if isPrefixDirectory {
-		return
-	}
 	err := lsNodeJSON(p.enc, path, node)
 	if err != nil {
 		Warnf("JSON encode failed: %v\n", err)
@@ -129,8 +121,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		AccessTime  time.Time   `json:"atime,omitempty"`
 		ChangeTime  time.Time   `json:"ctime,omitempty"`
 		Inode       uint64      `json:"inode,omitempty"`
-		MessageType string      `json:"message_type"` // "node"
+		StructType  string      `json:"struct_type"` // "node"
-		StructType  string      `json:"struct_type"`  // "node", deprecated
 
 		size uint64 // Target for Size pointer.
 	}{
@@ -146,7 +137,6 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		AccessTime:  node.AccessTime,
 		ChangeTime:  node.ChangeTime,
 		Inode:       node.Inode,
-		MessageType: "node",
 		StructType:  "node",
 	}
 	// Always print size for regular files, even when empty,
@@ -178,7 +168,7 @@ func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
 		Warnf("JSON encode failed: %v\n", err)
 	}
 	p.depth++
-	fmt.Fprintf(p.out, "[%d, %d, %s, [{\"name\":\"/\"}", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
+	fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
 }
 
 func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
@@ -196,13 +186,10 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
 		Mtime  int64  `json:"mtime"`
 	}
 
-	const blockSize = 512
-
 	outNode := NcduNode{
-		Name:  node.Name,
+		Name:   node.Name,
-		Asize: node.Size,
+		Asize:  node.Size,
-		// round up to nearest full blocksize
+		Dsize:  node.Size,
-		Dsize:  (node.Size + blockSize - 1) / blockSize * blockSize,
 		Dev:    node.DeviceID,
 		Ino:    node.Inode,
 		NLink:  node.Links,
@@ -222,15 +209,11 @@ func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
 	if node.Mode&os.ModeSticky != 0 {
 		outNode.Mode |= 0o1000
 	}
-	if outNode.Mtime < 0 {
-		// ncdu does not allow negative times
-		outNode.Mtime = 0
-	}
 
 	return json.Marshal(outNode)
 }
 
-func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) {
+func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
 	out, err := lsNcduNode(path, node)
 	if err != nil {
 		Warnf("JSON encode failed: %v\n", err)
@@ -250,7 +233,7 @@ func (p *ncduLsPrinter) LeaveDir(_ string) {
 }
 
 func (p *ncduLsPrinter) Close() {
-	fmt.Fprint(p.out, "\n]\n]\n")
+	fmt.Fprint(p.out, "\n]\n")
 }
 
 type textLsPrinter struct {
@@ -262,10 +245,8 @@ type textLsPrinter struct {
 func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
 	Verbosef("%v filtered by %v:\n", sn, p.dirs)
 }
-func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) {
+func (p *textLsPrinter) Node(path string, node *restic.Node) {
-	if !isPrefixDirectory {
+	Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
-		Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
-	}
 }
 
 func (p *textLsPrinter) LeaveDir(_ string) {}
@@ -324,11 +305,10 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return false
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
-	defer unlock()
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
@@ -382,11 +362,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 			return nil
 		}
 
-		printedDir := false
 		if withinDir(nodepath) {
-			// if we're within a target path, print the node
+			// if we're within a dir, print the node
-			printer.Node(nodepath, node, false)
+			printer.Node(nodepath, node)
-			printedDir = true
 
 			// if recursive listing is requested, signal the walker that it
 			// should continue walking recursively
@@ -398,20 +376,12 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		// if there's an upcoming match deeper in the tree (but we're not
 		// there yet), signal the walker to descend into any subdirs
 		if approachingMatchingTree(nodepath) {
-			// print node leading up to the target paths
-			if !printedDir {
-				printer.Node(nodepath, node, true)
-			}
 			return nil
 		}
 
 		// otherwise, signal the walker to not walk recursively into any
 		// subdirs
 		if node.Type == "dir" {
-			// immediately generate leaveDir if the directory is skipped
-			if printedDir {
-				printer.LeaveDir(nodepath)
-			}
 			return walker.ErrSkipNode
 		}
 		return nil
@@ -421,7 +391,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		ProcessNode: processNode,
 		LeaveDir: func(path string) {
 			// the root path `/` has no corresponding node and is thus also skipped by processNode
-			if path != "/" {
+			if withinDir(path) && path != "/" {
 				printer.LeaveDir(path)
 			}
 		},