Add version for 0.14.0

Fix typo in the environment variable name for `--from-password-file`

.dockerignore

@@ -0,0 +1,12 @@
+# Folders
+.git/
+.github/
+changelog/
+doc/
+docker/
+helpers/
+
+# Files
+.gitignore
+.golangci.yml
+*.md

.gitattributes

@@ -0,0 +1,2 @@
+# Workaround for https://github.com/golang/go/issues/52268.
+**/testdata/fuzz/*/* eol=lf

.github/workflows/tests.yml

@@ -8,6 +8,10 @@ on:
   # run tests for all pull requests
   pull_request:
 
+env:
+  latest_go: "1.19.x"
+  GO111MODULE: on
+
 jobs:
   test:
     strategy:
@@ -15,35 +19,47 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.16.x
+            go: 1.19.x
             os: windows-latest
+            install_verb: install
 
           - job_name: macOS
-            go: 1.16.x
+            go: 1.19.x
             os: macOS-latest
             test_fuse: false
+            install_verb: install
 
           - job_name: Linux
-            go: 1.16.x
+            go: 1.19.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
+            install_verb: install
+
+          - job_name: Linux
+            go: 1.18.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: install
+
+          - job_name: Linux
+            go: 1.17.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: install
+
+          - job_name: Linux
+            go: 1.16.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: get
 
           - job_name: Linux
             go: 1.15.x
             os: ubuntu-latest
             test_fuse: true
-
-          - job_name: Linux
-            go: 1.14.x
-            os: ubuntu-latest
-            test_fuse: true
-
-          - job_name: Linux
-            go: 1.13.x
-            os: ubuntu-latest
-            test_fuse: true
+            install_verb: get
 
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
@@ -60,7 +76,7 @@ jobs:
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go get github.com/restic/rest-server/...
+          go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -92,7 +108,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go get github.com/restic/rest-server/...
+          go ${{ matrix.install_verb }} github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -182,7 +198,7 @@ jobs:
       - name: Check changelog files with calens
         run: |
           echo "install calens"
-          go get github.com/restic/calens
+          go install github.com/restic/calens@latest
 
           echo "check changelog files"
           calens
@@ -206,7 +222,6 @@ jobs:
             solaris/amd64"
 
     env:
-      go: 1.16.x
       GOPROXY: https://proxy.golang.org
 
     runs-on: ubuntu-latest
@@ -214,17 +229,17 @@ jobs:
     name: Cross Compile for ${{ matrix.targets }}
 
     steps:
-      - name: Set up Go ${{ env.go }}
+      - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v2
         with:
-          go-version: ${{ env.go }}
-
-      - name: Check out code
-        uses: actions/checkout@v2
+          go-version: ${{ env.latest_go }}
 
       - name: Install gox
         run: |
-          go get github.com/mitchellh/gox
+          go install github.com/mitchellh/gox@latest
+
+      - name: Check out code
+        uses: actions/checkout@v2
 
       - name: Cross-compile with gox for ${{ matrix.targets }}
         env:
@@ -238,13 +253,11 @@ jobs:
   lint:
     name: lint
     runs-on: ubuntu-latest
-    env:
-      go: 1.16.x
     steps:
-      - name: Set up Go ${{ env.go }}
+      - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v2
         with:
-          go-version: ${{ env.go }}
+          go-version: ${{ env.latest_go }}
 
       - name: Check out code
         uses: actions/checkout@v2
@@ -253,7 +266,7 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.36
+          version: v1.48
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           only-new-issues: true
           args: --verbose --timeout 5m
@@ -269,3 +282,44 @@ jobs:
           echo "check if go.mod and go.sum are up to date"
           go mod tidy
           git diff --exit-code go.mod go.sum
+
+  docker:
+    name: docker
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            restic/restic
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: false
+          context: .
+          file: docker/Dockerfile
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.golangci.yml

@@ -24,7 +24,7 @@ linters:
     - govet
 
     # make sure names and comments are used according to the conventions
-    - golint
+    - revive
 
     # detect when assignments to existing variables are not used
     - ineffassign
@@ -51,7 +51,7 @@ issues:
 
   # list of things to not warn about
   exclude:
-    # golint: do not warn about missing comments for exported stuff
-    - exported (function|method|var|type|const) `.*` should have comment or be unexported
-    # golint: ignore constants in all caps
+    # revive: do not warn about missing comments for exported stuff
+    - exported (function|method|var|type|const) .* should have comment or be unexported
+    # revive: ignore constants in all caps
     - don't use ALL_CAPS in Go names; use CamelCase

CHANGELOG.md

@@ -1,3 +1,858 @@
+Changelog for restic 0.14.0 (2022-08-25)
+=======================================
+
+The following sections list the changes in restic 0.14.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2248: Support `self-update` on Windows
+ * Fix #3428: List snapshots in backend at most once to resolve snapshot IDs
+ * Fix #3432: Fix rare 'not found in repository' error for `copy` command
+ * Fix #3685: The `diff` command incorrectly listed some files as added
+ * Fix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows
+ * Fix #3720: Directory sync errors for repositories accessed via SMB
+ * Fix #3736: The `stats` command miscalculated restore size for multiple snapshots
+ * Fix #3861: Yield error on invalid policy to `forget`
+ * Fix #3716: Print "wrong password" to stderr instead of stdout
+ * Fix #3772: Correctly rebuild index for legacy repositories
+ * Fix #3776: Limit number of key files tested while opening a repository
+ * Chg #1842: Support debug log creation in release builds
+ * Chg #3295: Deprecate `check --check-unused` and add further checks
+ * Chg #3680: Update dependencies and require Go 1.15 or newer
+ * Chg #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
+ * Enh #1153: Support pruning even when the disk is full
+ * Enh #21: Add compression support
+ * Enh #2162: Adaptive IO concurrency based on backend connections
+ * Enh #2291: Allow pack size customization
+ * Enh #2295: Allow use of SAS token to authenticate to Azure
+ * Enh #2696: Improve backup speed with many small files
+ * Enh #2907: Make snapshot directory structure of `mount` command customizable
+ * Enh #3114: Optimize handling of duplicate blobs in `prune`
+ * Enh #3465: Improve handling of temporary files on Windows
+ * Enh #3709: Validate exclude patterns before backing up
+ * Enh #3837: Improve SFTP repository initialization over slow links
+ * Enh #2351: Use config file permissions to control file group access
+ * Enh #3475: Allow limiting IO concurrency for local and SFTP backend
+ * Enh #3484: Stream data in `check` and `prune` commands
+ * Enh #2923: Improve speed of `copy` command
+ * Enh #3729: Display full IDs in `check` warnings
+ * Enh #3773: Optimize memory usage for directories with many files
+ * Enh #3819: Validate include/exclude patterns before restoring
+
+Details
+-------
+
+ * Bugfix #2248: Support `self-update` on Windows
+
+   Restic `self-update` would fail in situations where the operating system locks running
+   binaries, including Windows. The new behavior works around this by renaming the running file
+   and swapping the updated file in place.
+
+   https://github.com/restic/restic/issues/2248
+   https://github.com/restic/restic/pull/3675
+
+ * Bugfix #3428: List snapshots in backend at most once to resolve snapshot IDs
+
+   Many commands support specifying a list of snapshot IDs which are then used to determine the
+   snapshots to be processed by the command. To resolve snapshot IDs or `latest`, and check that
+   these exist, restic previously listed all snapshots stored in the repository. Depending on
+   the backend this could be a slow and/or expensive operation.
+
+   Restic now lists the snapshots only once and remembers the result in order to resolve all
+   further snapshot IDs swiftly.
+
+   https://github.com/restic/restic/issues/3428
+   https://github.com/restic/restic/pull/3570
+   https://github.com/restic/restic/pull/3395
+
+ * Bugfix #3432: Fix rare 'not found in repository' error for `copy` command
+
+   In rare cases `copy` (and other commands) would report that `LoadTree(...)` returned an `id
+   [...] not found in repository` error. This could be caused by a backup or copy command running
+   concurrently. The error was only temporary; running the failed restic command a second time as
+   a workaround did resolve the error.
+
+   This issue has now been fixed by correcting the order in which restic reads data from the
+   repository. It is now guaranteed that restic only loads snapshots for which all necessary data
+   is already available.
+
+   https://github.com/restic/restic/issues/3432
+   https://github.com/restic/restic/pull/3570
+
+ * Bugfix #3685: The `diff` command incorrectly listed some files as added
+
+   There was a bug in the `diff` command, causing it to always show files in a removed directory as
+   added. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3685
+   https://github.com/restic/restic/pull/3686
+
+ * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows
+
+   In #3602 a fix was introduced to address the problem of `rclone` prematurely exiting when
+   Ctrl+C is pressed on Windows. The solution was to create the subprocess with its console
+   detached from the restic console.
+
+   However, this solution failed when using `rclone` installed by Scoop or using `sftp` with a
+   passphrase-protected private key. We've now fixed this by using a different approach to
+   prevent Ctrl-C from passing down too early.
+
+   https://github.com/restic/restic/issues/3681
+   https://github.com/restic/restic/issues/3692
+   https://github.com/restic/restic/pull/3696
+
+ * Bugfix #3720: Directory sync errors for repositories accessed via SMB
+
+   On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in restic failing to
+   save the lock file, yielding the following errors:
+
+   Save(<lock/071fe833f0>) returned error, retrying after 552.330144ms: sync /repo/locks:
+   no such file or directory Save(<lock/bf789d7343>) returned error, retrying after
+   552.330144ms: sync /repo/locks: invalid argument
+
+   This has now been fixed by ignoring the relevant error codes.
+
+   https://github.com/restic/restic/issues/3720
+   https://github.com/restic/restic/issues/3751
+   https://github.com/restic/restic/pull/3752
+
+ * Bugfix #3736: The `stats` command miscalculated restore size for multiple snapshots
+
+   Since restic 0.10.0 the restore size calculated by the `stats` command for multiple snapshots
+   was too low. The hardlink detection was accidentally applied across multiple snapshots and
+   thus ignored many files. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3736
+   https://github.com/restic/restic/pull/3740
+
+ * Bugfix #3861: Yield error on invalid policy to `forget`
+
+   The `forget` command previously silently ignored invalid/unsupported units in the duration
+   options, such as e.g. `--keep-within-daily 2w`.
+
+   Specifying an invalid/unsupported duration unit now results in an error.
+
+   https://github.com/restic/restic/issues/3861
+   https://github.com/restic/restic/pull/3862
+
+ * Bugfix #3716: Print "wrong password" to stderr instead of stdout
+
+   If an invalid password was entered, the error message was printed on stdout and not on stderr as
+   intended. This has now been fixed.
+
+   https://github.com/restic/restic/pull/3716
+   https://forum.restic.net/t/4965
+
+ * Bugfix #3772: Correctly rebuild index for legacy repositories
+
+   After running `rebuild-index` on a legacy repository containing mixed pack files (that is,
+   pack files which store both metadata and file data), `check` printed warnings like `pack
+   12345678 contained in several indexes: ...`. This warning was not critical, but has now
+   nonetheless been fixed by properly handling mixed pack files while rebuilding the index.
+
+   Running `prune` for such legacy repositories will also fix the warning by reorganizing the
+   pack files which caused it.
+
+   https://github.com/restic/restic/pull/3772
+   https://github.com/restic/restic/pull/3884
+   https://forum.restic.net/t/5044/13
+
+ * Bugfix #3776: Limit number of key files tested while opening a repository
+
+   Previously, restic tested the password against every key in the repository when opening a
+   repository. The more keys there were in the repository, the slower this operation became.
+
+   Restic now tests the password against up to 20 key files in the repository. Alternatively, you
+   can use the `--key-hint=<key ID>` option to specify a specific key file to use instead.
+
+   https://github.com/restic/restic/pull/3776
+
+ * Change #1842: Support debug log creation in release builds
+
+   Creating a debug log was only possible in debug builds which required users to manually build
+   restic. We changed the release builds to allow creating debug logs by simply setting the
+   environment variable `DEBUG_LOG=logname.log`.
+
+   https://github.com/restic/restic/issues/1842
+   https://github.com/restic/restic/pull/3826
+
+ * Change #3295: Deprecate `check --check-unused` and add further checks
+
+   Since restic 0.12.0, it is expected to still have unused blobs after running `prune`. This made
+   the `--check-unused` option of the `check` command rather useless and tended to confuse
+   users. This option has been deprecated and is now ignored.
+
+   The `check` command now also warns if a repository is using either the legacy S3 layout or mixed
+   pack files with both tree and data blobs. The latter is known to cause performance problems.
+
+   https://github.com/restic/restic/issues/3295
+   https://github.com/restic/restic/pull/3730
+
+ * Change #3680: Update dependencies and require Go 1.15 or newer
+
+   We've updated most dependencies. Since some libraries require newer language features we're
+   dropping support for Go 1.14, which means that restic now requires at least Go 1.15 to build.
+
+   https://github.com/restic/restic/issues/3680
+   https://github.com/restic/restic/issues/3883
+
+ * Change #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
+
+   The `init` and `copy` commands can read data from another repository. However, confusingly
+   `--repo2` referred to the repository *from* which the `init` command copies parameters, but
+   for the `copy` command `--repo2` referred to the copy *destination*.
+
+   We've introduced a new option, `--from-repo`, which always refers to the source repository
+   for both commands. The old parameter names have been deprecated but still work. To create a new
+   repository and copy all snapshots to it, the commands are now as follows:
+
+   ``` restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo
+   --copy-chunker-params restic -r /srv/restic-repo-copy copy --from-repo
+   /srv/restic-repo ```
+
+   https://github.com/restic/restic/pull/3742
+   https://forum.restic.net/t/5017
+
+ * Enhancement #1153: Support pruning even when the disk is full
+
+   When running out of disk space it was no longer possible to add or remove data from a repository.
+   To help with recovering from such a deadlock, the prune command now supports an
+   `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to
+   read the documentation first!
+
+   https://github.com/restic/restic/issues/1153
+   https://github.com/restic/restic/pull/3481
+
+ * Enhancement #21: Add compression support
+
+   We've added compression support to the restic repository format. To create a repository using
+   the new format run `init --repository-version 2`. Please note that the repository cannot be
+   read by restic versions prior to 0.14.0.
+
+   You can configure whether data is compressed with the option `--compression`. It can be set to
+   `auto` (the default, which will compress very fast), `max` (which will trade backup speed and
+   CPU usage for better compression), or `off` (which disables compression). Each setting is
+   only applied for the current run of restic and does *not* apply to future runs. The option can
+   also be set via the environment variable `RESTIC_COMPRESSION`.
+
+   To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the documentation
+   for more details. The migration checks the repository integrity and upgrades the repository
+   format, but will not change any data. Afterwards, prune will rewrite the metadata to make use of
+   compression.
+
+   As an alternative you can use the `copy` command to migrate snapshots; First create a new
+   repository using `init --repository-version 2 --copy-chunker-params --repo2
+   path/to/old/repo`, and then use the `copy` command to copy all snapshots to the new
+   repository.
+
+   https://github.com/restic/restic/issues/21
+   https://github.com/restic/restic/issues/3779
+   https://github.com/restic/restic/pull/3666
+   https://github.com/restic/restic/pull/3704
+   https://github.com/restic/restic/pull/3733
+
+ * Enhancement #2162: Adaptive IO concurrency based on backend connections
+
+   Many commands used hard-coded limits for the number of concurrent operations. This prevented
+   speed improvements by increasing the number of connections used by a backend.
+
+   These limits have now been replaced by using the configured number of backend connections
+   instead, which can be controlled using the `-o <backend-name>.connections=5` option.
+   Commands will then automatically scale their parallelism accordingly.
+
+   To limit the number of CPU cores used by restic, you can set the environment variable
+   `GOMAXPROCS` accordingly. For example to use a single CPU core, use `GOMAXPROCS=1`.
+
+   https://github.com/restic/restic/issues/2162
+   https://github.com/restic/restic/issues/1467
+   https://github.com/restic/restic/pull/3611
+
+ * Enhancement #2291: Allow pack size customization
+
+   Restic now uses a target pack size of 16 MiB by default. This can be customized using the
+   `--pack-size size` option. Supported pack sizes range between 4 and 128 MiB.
+
+   It is possible to migrate an existing repository to _larger_ pack files using `prune
+   --repack-small`. This will rewrite every pack file which is significantly smaller than the
+   target size.
+
+   https://github.com/restic/restic/issues/2291
+   https://github.com/restic/restic/pull/3731
+
+ * Enhancement #2295: Allow use of SAS token to authenticate to Azure
+
+   Previously restic only supported AccountKeys to authenticate to Azure storage accounts,
+   which necessitates giving a significant amount of access.
+
+   We added support for Azure SAS tokens which are a more fine-grained and time-limited manner of
+   granting access. Set the `AZURE_ACCOUNT_NAME` and `AZURE_ACCOUNT_SAS` environment
+   variables to use a SAS token for authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it
+   will take precedence.
+
+   https://github.com/restic/restic/issues/2295
+   https://github.com/restic/restic/pull/3661
+
+ * Enhancement #2696: Improve backup speed with many small files
+
+   We have restructured the backup pipeline to continue reading files while all upload
+   connections are busy. This allows the backup to already prepare the next data file such that the
+   upload can continue as soon as a connection becomes available. This can especially improve the
+   backup performance for high latency backends.
+
+   The upload concurrency is now controlled using the `-o <backend-name>.connections=5`
+   option.
+
+   https://github.com/restic/restic/issues/2696
+   https://github.com/restic/restic/pull/3489
+
+ * Enhancement #2907: Make snapshot directory structure of `mount` command customizable
+
+   We've added the possibility to customize the snapshot directory structure of the `mount`
+   command using templates passed to the `--snapshot-template` option. The formatting of
+   snapshots' timestamps is now controlled using `--time-template` and supports
+   subdirectories to for example group snapshots by year. Please see `restic help mount` for
+   further details.
+
+   Characters in tag names which are not allowed in a filename are replaced by underscores `_`. For
+   example a tag `foo/bar` will result in a directory name of `foo_bar`.
+
+   https://github.com/restic/restic/issues/2907
+   https://github.com/restic/restic/pull/2913
+   https://github.com/restic/restic/pull/3691
+
+ * Enhancement #3114: Optimize handling of duplicate blobs in `prune`
+
+   Restic `prune` always used to repack all data files containing duplicate blobs. This
+   effectively removed all duplicates during prune. However, as a consequence all these data
+   files were repacked even if the unused repository space threshold could be reached with less
+   work.
+
+   This is now changed and `prune` works nice and fast even when there are lots of duplicate blobs.
+
+   https://github.com/restic/restic/issues/3114
+   https://github.com/restic/restic/pull/3290
+
+ * Enhancement #3465: Improve handling of temporary files on Windows
+
+   In some cases restic failed to delete temporary files, causing the current command to fail.
+   This has now been fixed by ensuring that Windows automatically deletes the file. In addition,
+   temporary files are only written to disk when necessary, reducing disk writes.
+
+   https://github.com/restic/restic/issues/3465
+   https://github.com/restic/restic/issues/1551
+   https://github.com/restic/restic/pull/3610
+
+ * Enhancement #3709: Validate exclude patterns before backing up
+
+   Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or
+   `--iexclude-file` previously weren't validated. As a consequence, invalid patterns
+   resulted in files that were meant to be excluded being backed up.
+
+   Restic now validates all patterns before running the backup and aborts with a fatal error if an
+   invalid pattern is detected.
+
+   https://github.com/restic/restic/issues/3709
+   https://github.com/restic/restic/pull/3734
+
+ * Enhancement #3837: Improve SFTP repository initialization over slow links
+
+   The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the
+   backend concurrently. This reduces the waiting times when creating a repository over a very
+   slow connection.
+
+   https://github.com/restic/restic/issues/3837
+   https://github.com/restic/restic/pull/3840
+
+ * Enhancement #2351: Use config file permissions to control file group access
+
+   Previously files in a local/SFTP repository would always end up with very restrictive access
+   permissions, allowing access only to the owner. This prevented a number of valid use-cases
+   involving groups and ACLs.
+
+   We now use the permissions of the config file in the repository to decide whether group access
+   should be given to newly created repository files or not. We arrange for repository files to be
+   created group readable exactly when the repository config file is group readable.
+
+   To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config
+   file can be used. For repositories that should be writable by group members a tad more setup is
+   required, see the docs.
+
+   Posix ACLs can also be used now that the group permissions being forced to zero no longer masks
+   the effect of ACL entries.
+
+   https://github.com/restic/restic/issues/2351
+   https://github.com/restic/restic/pull/3419
+   https://forum.restic.net/t/1391
+
+ * Enhancement #3475: Allow limiting IO concurrency for local and SFTP backend
+
+   Restic did not support limiting the IO concurrency / number of connections for accessing
+   repositories stored using the local or SFTP backends. The number of connections is now limited
+   as for other backends, and can be configured via the the `-o local.connections=2` and `-o
+   sftp.connections=5` options. This ensures that restic does not overwhelm the backend with
+   concurrent IO operations.
+
+   https://github.com/restic/restic/pull/3475
+
+ * Enhancement #3484: Stream data in `check` and `prune` commands
+
+   The commands `check --read-data` and `prune` previously downloaded data files into
+   temporary files which could end up being written to disk. This could cause a large amount of data
+   being written to disk.
+
+   The pack files are now instead streamed, which removes the need for temporary files. Please
+   note that *uploads* during `backup` and `prune` still require temporary files.
+
+   https://github.com/restic/restic/issues/3710
+   https://github.com/restic/restic/pull/3484
+   https://github.com/restic/restic/pull/3717
+
+ * Enhancement #2923: Improve speed of `copy` command
+
+   The `copy` command could require a long time to copy snapshots for non-local backends. This has
+   been improved to provide a throughput comparable to the `restore` command.
+
+   Additionally, `copy` now displays a progress bar.
+
+   https://github.com/restic/restic/issues/2923
+   https://github.com/restic/restic/pull/3513
+
+ * Enhancement #3729: Display full IDs in `check` warnings
+
+   When running commands to inspect or repair a damaged repository, it is often necessary to
+   supply the full IDs of objects stored in the repository.
+
+   The output of `check` now includes full IDs instead of their shortened variant.
+
+   https://github.com/restic/restic/pull/3729
+
+ * Enhancement #3773: Optimize memory usage for directories with many files
+
+   Backing up a directory with hundreds of thousands or more files caused restic to require large
+   amounts of memory. We've now optimized the `backup` command such that it requires up to 30% less
+   memory.
+
+   https://github.com/restic/restic/pull/3773
+
+ * Enhancement #3819: Validate include/exclude patterns before restoring
+
+   Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include` and
+   `--iinclude` weren't validated before running the restore. Invalid patterns would result in
+   error messages being printed repeatedly, and possibly unwanted files being restored.
+
+   Restic now validates all patterns before running the restore, and aborts with a fatal error if
+   an invalid pattern is detected.
+
+   https://github.com/restic/restic/pull/3819
+
+
+Changelog for restic 0.13.0 (2022-03-26)
+=======================================
+
+The following sections list the changes in restic 0.13.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1106: Never lock repository for `list locks`
+ * Fix #2345: Make cache crash-resistant and usable by multiple concurrent processes
+ * Fix #2452: Improve error handling of repository locking
+ * Fix #2738: Don't print progress for `backup --json --quiet`
+ * Fix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+ * Fix #3518: Make `copy` command honor `--no-lock` for source repository
+ * Fix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
+ * Fix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+ * Fix #3667: The `mount` command now reports symlinks sizes
+ * Fix #3488: `rebuild-index` failed if an index file was damaged
+ * Fix #3591: Fix handling of `prune --max-repack-size=0`
+ * Fix #3619: Avoid choosing parent snapshots newer than time of new snapshot
+ * Chg #3641: Ignore parent snapshot for `backup --stdin`
+ * Chg #3519: Require Go 1.14 or newer
+ * Enh #1542: Add `--dry-run`/`-n` option to `backup` command
+ * Enh #2202: Add upload checksum for Azure, GS, S3 and Swift backends
+ * Enh #233: Support negative include/exclude patterns
+ * Enh #2388: Add warning for S3 if partial credentials are provided
+ * Enh #2508: Support JSON output and quiet mode for the `diff` command
+ * Enh #2656: Add flag to disable TLS verification for self-signed certificates
+ * Enh #3003: Atomic uploads for the SFTP backend
+ * Enh #3127: Add xattr (extended attributes) support for Solaris
+ * Enh #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+ * Enh #3490: Support random subset by size in `check --read-data-subset`
+ * Enh #3541: Improve handling of temporary B2 delete errors
+ * Enh #3542: Add file mode in symbolic notation to `ls --json`
+ * Enh #2594: Speed up the `restore --verify` command
+ * Enh #2816: The `backup` command no longer updates file access times on Linux
+ * Enh #2880: Make `recover` collect only unreferenced trees
+ * Enh #3429: Verify that new or modified keys are stored correctly
+ * Enh #3436: Improve local backend's resilience to (system) crashes
+ * Enh #3508: Cache blobs read by the `dump` command
+ * Enh #3511: Support configurable timeout for the rclone backend
+ * Enh #3593: Improve `copy` performance by parallelizing IO
+
+Details
+-------
+
+ * Bugfix #1106: Never lock repository for `list locks`
+
+   The `list locks` command previously locked to the repository by default. This had the problem
+   that it wouldn't work for an exclusively locked repository and that the command would also
+   display its own lock file which can be confusing.
+
+   Now, the `list locks` command never locks the repository.
+
+   https://github.com/restic/restic/issues/1106
+   https://github.com/restic/restic/pull/3665
+
+ * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes
+
+   The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of
+   restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a
+   temporary location before being moved to their proper location.
+
+   This also allows multiple concurrent restic processes to operate on a single repository
+   without conflicts. Previously, concurrent operations could cause segfaults because the
+   processes saw each other's partially downloaded files.
+
+   https://github.com/restic/restic/issues/2345
+   https://github.com/restic/restic/pull/2838
+
+ * Bugfix #2452: Improve error handling of repository locking
+
+   Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly
+   created one. Instead it continued trying to delete the old (usually no longer existing) lock
+   file and thus over time lots of lock files accumulated. This has now been fixed.
+
+   https://github.com/restic/restic/issues/2452
+   https://github.com/restic/restic/issues/2473
+   https://github.com/restic/restic/issues/2562
+   https://github.com/restic/restic/pull/3512
+
+ * Bugfix #2738: Don't print progress for `backup --json --quiet`
+
+   Unlike the text output, the `--json` output format still printed progress information even in
+   `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode.
+
+   https://github.com/restic/restic/issues/2738
+   https://github.com/restic/restic/pull/3264
+
+ * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+
+   Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable,
+   which caused problems in certain system/usage configurations. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3382
+   https://github.com/restic/restic/pull/3474
+
+ * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository
+
+   The `copy` command previously did not respect the `--no-lock` option for the source
+   repository, causing failures with read-only storage backends. This has now been fixed such
+   that the option is now respected.
+
+   https://github.com/restic/restic/issues/3518
+   https://github.com/restic/restic/pull/3589
+
+ * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
+
+   Previously, if a request failed with an SSL unknown certificate authority error, the B2
+   backend retried indefinitely and restic would appear to hang.
+
+   This has now been fixed and restic instead fails with an error message.
+
+   https://github.com/restic/restic/issues/3556
+   https://github.com/restic/restic/issues/2355
+   https://github.com/restic/restic/pull/3571
+
+ * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+
+   Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the
+   backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time
+   as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone
+   stdio connection already closed" errors.
+
+   This has now been fixed by restic starting the rclone process detached from the console restic
+   runs in (similar to starting processes in a new process group on Linux), which enables restic to
+   gracefully clean up rclone (which now never gets the `SIGINT`).
+
+   https://github.com/restic/restic/issues/3601
+   https://github.com/restic/restic/pull/3602
+
+ * Bugfix #3667: The `mount` command now reports symlinks sizes
+
+   Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They
+   now have a size equal to the byte length of their target path, as required by POSIX.
+
+   https://github.com/restic/restic/issues/3667
+   https://github.com/restic/restic/pull/3668
+
+ * Bugfix #3488: `rebuild-index` failed if an index file was damaged
+
+   Previously, the `rebuild-index` command would fail with an error if an index file was damaged
+   or truncated. This has now been fixed.
+
+   On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or
+   to manually delete the damaged index.
+
+   https://github.com/restic/restic/pull/3488
+
+ * Bugfix #3591: Fix handling of `prune --max-repack-size=0`
+
+   Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been
+   fixed.
+
+   As a workaround, `--max-repack-size=1` can be used with older versions of restic.
+
+   https://github.com/restic/restic/pull/3591
+
+ * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot
+
+   The `backup` command, when a `--parent` was not provided, previously chose the most recent
+   matching snapshot as the parent snapshot. However, this didn't make sense when the user passed
+   `--time` to create a new snapshot older than the most recent snapshot.
+
+   Instead, `backup` now chooses the most recent snapshot which is not newer than the
+   snapshot-being-created's timestamp, to avoid any time travel.
+
+   https://github.com/restic/restic/pull/3619
+
+ * Change #3641: Ignore parent snapshot for `backup --stdin`
+
+   Restic uses a parent snapshot to speed up directory scanning when performing backups, but this
+   only wasted time and memory when the backup source is stdin (using the `--stdin` option of the
+   `backup` command), since no directory scanning is performed in this case.
+
+   Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to
+   skip some startup operations and saves a bit of resources.
+
+   The `--parent` option is still available for `backup --stdin`, but is now ignored.
+
+   https://github.com/restic/restic/issues/3641
+   https://github.com/restic/restic/pull/3645
+
+ * Change #3519: Require Go 1.14 or newer
+
+   Restic now requires Go 1.14 to build. This allows it to use new standard library features
+   instead of an external dependency.
+
+   https://github.com/restic/restic/issues/3519
+
+ * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command
+
+   Testing exclude filters and other configuration options was error prone as wrong filters
+   could cause files to be uploaded unintentionally. It was also not possible to estimate
+   beforehand how much data would be uploaded.
+
+   The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of
+   a backup without actually writing anything to the repository.
+
+   Passing -vv will log information about files that would be added, allowing for verification of
+   source and exclusion options before running the real backup.
+
+   https://github.com/restic/restic/issues/1542
+   https://github.com/restic/restic/pull/2308
+   https://github.com/restic/restic/pull/3210
+   https://github.com/restic/restic/pull/3300
+
+ * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends
+
+   Previously only the B2 and partially the Swift backends verified the integrity of uploaded
+   (encrypted) files. The verification works by informing the backend about the expected hash of
+   the uploaded file. The backend then verifies the upload and thereby rules out any data
+   corruption during upload.
+
+   We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides
+   integrity checking for uploads also means that restic can now be used to store backups in S3
+   buckets which have Object Lock enabled.
+
+   https://github.com/restic/restic/issues/2202
+   https://github.com/restic/restic/issues/2700
+   https://github.com/restic/restic/issues/3023
+   https://github.com/restic/restic/pull/3246
+
+ * Enhancement #233: Support negative include/exclude patterns
+
+   If a pattern starts with an exclamation mark and it matches a file that was previously matched by
+   a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to
+   cancel the exclusion of some files.
+
+   It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it
+   is not possible to include files inside the directory.
+
+   Example of use as an exclude pattern for the `backup` command:
+
+   $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...]
+   node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...]
+
+   https://github.com/restic/restic/issues/233
+   https://github.com/restic/restic/pull/2311
+
+ * Enhancement #2388: Add warning for S3 if partial credentials are provided
+
+   Previously restic did not notify about incomplete credentials when using the S3 backend,
+   instead just reporting access denied.
+
+   Restic now checks that both the AWS key ID and secret environment variables are set before
+   connecting to the remote server, and reports an error if not.
+
+   https://github.com/restic/restic/issues/2388
+   https://github.com/restic/restic/pull/3532
+
+ * Enhancement #2508: Support JSON output and quiet mode for the `diff` command
+
+   The `diff` command now supports outputting machine-readable output in JSON format. To enable
+   this, pass the `--json` option to the command. To only print the summary and suppress detailed
+   output, pass the `--quiet` option.
+
+   https://github.com/restic/restic/issues/2508
+   https://github.com/restic/restic/pull/3592
+
+ * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates
+
+   There is now an `--insecure-tls` global option in restic, which disables TLS verification for
+   self-signed certificates in order to support some development workflows.
+
+   https://github.com/restic/restic/issues/2656
+   https://github.com/restic/restic/pull/2657
+
+ * Enhancement #3003: Atomic uploads for the SFTP backend
+
+   The SFTP backend did not upload files atomically. An interrupted upload could leave an
+   incomplete file behind which could prevent restic from accessing the repository. This has now
+   been fixed and uploads in the SFTP backend are done atomically.
+
+   https://github.com/restic/restic/issues/3003
+   https://github.com/restic/restic/pull/3524
+
+ * Enhancement #3127: Add xattr (extended attributes) support for Solaris
+
+   Restic now supports xattr for the Solaris operating system.
+
+   https://github.com/restic/restic/issues/3127
+   https://github.com/restic/restic/pull/3628
+
+ * Enhancement #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+
+   Restic used to silently ignore the `--no-lock` option of the `forget` command.
+
+   It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If
+   `--no-lock` option is specified without `--dry-run`, restic prints a warning message to
+   stderr.
+
+   https://github.com/restic/restic/issues/3464
+   https://github.com/restic/restic/pull/3623
+
+ * Enhancement #3490: Support random subset by size in `check --read-data-subset`
+
+   The `--read-data-subset` option of the `check` command now supports a third way of specifying
+   the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or
+   t/T.
+
+   https://github.com/restic/restic/issues/3490
+   https://github.com/restic/restic/pull/3548
+
+ * Enhancement #3541: Improve handling of temporary B2 delete errors
+
+   Deleting files on B2 could sometimes fail temporarily, which required restic to retry the
+   delete operation. In some cases the file was deleted nevertheless, causing the retries and
+   ultimately the restic command to fail. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3541
+   https://github.com/restic/restic/pull/3544
+
+ * Enhancement #3542: Add file mode in symbolic notation to `ls --json`
+
+   The `ls --json` command now provides the file mode in symbolic notation (using the
+   `permissions` key), aligned with `find --json`.
+
+   https://github.com/restic/restic/issues/3542
+   https://github.com/restic/restic/pull/3573
+   https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
+
+ * Enhancement #2594: Speed up the `restore --verify` command
+
+   The `--verify` option lets the `restore` command verify the file content after it has restored
+   a snapshot. The performance of this operation has now been improved by up to a factor of two.
+
+   https://github.com/restic/restic/pull/2594
+
+ * Enhancement #2816: The `backup` command no longer updates file access times on Linux
+
+   When reading files during backup, restic used to cause the operating system to update the
+   files' access times. Note that this did not apply to filesystems with disabled file access
+   times.
+
+   Restic now instructs the operating system not to update the file access time, if the user
+   running restic is the file owner or has root permissions.
+
+   https://github.com/restic/restic/pull/2816
+
+ * Enhancement #2880: Make `recover` collect only unreferenced trees
+
+   Previously, the `recover` command used to generate a snapshot containing *all* root trees,
+   even those which were already referenced by a snapshot.
+
+   This has been improved such that it now only processes trees not already referenced by any
+   snapshot.
+
+   https://github.com/restic/restic/pull/2880
+
+ * Enhancement #3429: Verify that new or modified keys are stored correctly
+
+   When adding a new key or changing the password of a key, restic used to just create the new key (and
+   remove the old one, when changing the password). There was no verification that the new key was
+   stored correctly and works properly. As the repository cannot be decrypted without a valid key
+   file, this could in rare cases cause the repository to become inaccessible.
+
+   Restic now checks that new key files actually work before continuing. This can protect against
+   some (rare) cases of hardware or storage problems.
+
+   https://github.com/restic/restic/pull/3429
+
+ * Enhancement #3436: Improve local backend's resilience to (system) crashes
+
+   Restic now ensures that files stored using the `local` backend are created atomically (that
+   is, files are either stored completely or not at all). This ensures that no incomplete files are
+   left behind even if restic is terminated while writing a file.
+
+   In addition, restic now tries to ensure that the directory in the repository which contains a
+   newly uploaded file is also written to disk. This can prevent missing files if the system
+   crashes or the disk is not properly unmounted.
+
+   https://github.com/restic/restic/pull/3436
+
+ * Enhancement #3508: Cache blobs read by the `dump` command
+
+   When dumping a file using the `dump` command, restic did not cache blobs in any way, so even
+   consecutive runs of the same blob were loaded from the repository again and again, slowing down
+   the dump.
+
+   Now, the caching mechanism already used by the `fuse` command is also used by the `dump`
+   command. This makes dumping much faster, especially for sparse files.
+
+   https://github.com/restic/restic/pull/3508
+
+ * Enhancement #3511: Support configurable timeout for the rclone backend
+
+   A slow rclone backend could cause restic to time out while waiting for the repository to open.
+   Restic now offers an `-o rclone.timeout` option to make this timeout configurable.
+
+   https://github.com/restic/restic/issues/3511
+   https://github.com/restic/restic/pull/3514
+
+ * Enhancement #3593: Improve `copy` performance by parallelizing IO
+
+   Restic copy previously only used a single thread for copying blobs between repositories,
+   which resulted in limited performance when copying small blobs to/from a high latency backend
+   (i.e. any remote backend, especially b2).
+
+   Copying will now use 8 parallel threads to increase the throughput of the copy operation.
+
+   https://github.com/restic/restic/pull/3593
+
+
 Changelog for restic 0.12.1 (2021-08-03)
 =======================================
 
@@ -3083,7 +3938,7 @@ Summary
  * Enh #1055: Create subdirs below `data/` for local/sftp backends
  * Enh #1067: Allow loading credentials for s3 from IAM
  * Enh #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
- * Enh #1081: Clarify semantic for `--tasg` for the `forget` command
+ * Enh #1081: Clarify semantic for `--tag` for the `forget` command
  * Enh #1080: Ignore chmod() errors on filesystems which do not support it
  * Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t)
 
@@ -3127,7 +3982,7 @@ Details
    https://github.com/restic/restic/issues/1073
    https://github.com/restic/restic/pull/1075
 
- * Enhancement #1081: Clarify semantic for `--tasg` for the `forget` command
+ * Enhancement #1081: Clarify semantic for `--tag` for the `forget` command
 
    https://github.com/restic/restic/issues/1081
    https://github.com/restic/restic/pull/1090

CONTRIBUTING.md

@@ -48,9 +48,8 @@ environment was used and so on. Please tell us at least the following things:
 Remember, the easier it is for us to reproduce the bug, the earlier it will be
 corrected!
 
-In addition, you can compile restic with debug support by running
-`go run build.go -tags debug` and instructing it to create a debug
-log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
+In addition, you can instruct restic to create a debug log by setting the
+environment variable `DEBUG_LOG` to a file, e.g. like this:
 
     $ export DEBUG_LOG=/tmp/restic-debug.log
     $ restic backup ~/work
@@ -66,8 +65,8 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.13 or later for compiling. Clone the repo (without
-having `$GOPATH` set) and `cd` into the directory:
+Make sure you have the minimum required Go version installed. Clone the repo
+(without having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH
     $ git clone https://github.com/restic/restic
@@ -123,7 +122,10 @@ down to the following steps:
     writing, ask yourself: If I were the user, what would I need to be aware
     of with this change?
 
- 8. Once your code looks good and passes all the tests, we'll merge it. Thanks
+ 8. Do not edit the man pages under `doc/man` or `doc/manual_rest.rst` -
+    these are autogenerated before new releases.
+
+ 9. Once your code looks good and passes all the tests, we'll merge it. Thanks
     a lot for your contribution!
 
 Please provide the patches for each bug or feature in a separate branch and

README.md

@@ -46,8 +46,8 @@ Therefore, restic supports the following backends for storing backups natively:
 
 - [Local directory](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local)
 - [sftp server (via SSH)](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp)
-- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](doc/100_references.rst#rest-backend), [rest-server](https://github.com/restic/rest-server))
-- [AWS S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
+- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](https://restic.readthedocs.io/en/latest/100_references.html#rest-backend), [rest-server](https://github.com/restic/rest-server))
+- [Amazon S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
 - [OpenStack Swift](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift)
 - [BackBlaze B2](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2)
 - [Microsoft Azure Blob Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)

VERSION

@@ -1 +1 @@
-0.12.1
+0.14.0

build.go

@@ -35,6 +35,7 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+//go:build ignore_build_go
 // +build ignore_build_go
 
 package main
@@ -58,7 +59,7 @@ var config = Config{
 	Main:             "./cmd/restic",                           // package name for the main package
 	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
 	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 11, Patch: 0}, // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
@@ -123,17 +124,8 @@ func printEnv(env []string) {
 
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd string, env map[string]string, args ...string) error {
-	a := []string{"build"}
-
-	// try to remove all absolute paths from resulting binary
-	if goVersion.AtLeast(GoVersion{1, 13, 0}) {
-		// use the new flag introduced by Go 1.13
-		a = append(a, "-trimpath")
-	} else {
-		// otherwise try to trim as many paths as possible
-		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-	}
+	// -trimpath removes all absolute paths from the binary.
+	a := []string{"build", "-trimpath"}
 
 	if enablePIE {
 		a = append(a, "-buildmode=pie")

changelog/0.13.0_2022-03-26/issue-1106

@@ -0,0 +1,10 @@
+Bugfix: Never lock repository for `list locks`
+
+The `list locks` command previously locked to the repository by default. This
+had the problem that it wouldn't work for an exclusively locked repository and
+that the command would also display its own lock file which can be confusing.
+
+Now, the `list locks` command never locks the repository.
+
+https://github.com/restic/restic/issues/1106
+https://github.com/restic/restic/pull/3665

changelog/0.13.0_2022-03-26/issue-1542

@@ -0,0 +1,16 @@
+Enhancement: Add `--dry-run`/`-n` option to `backup` command
+
+Testing exclude filters and other configuration options was error prone as
+wrong filters could cause files to be uploaded unintentionally. It was also
+not possible to estimate beforehand how much data would be uploaded.
+
+The `backup` command now has a `--dry-run`/`-n` option, which performs all the
+normal steps of a backup without actually writing anything to the repository.
+
+Passing -vv will log information about files that would be added, allowing for
+verification of source and exclusion options before running the real backup.
+
+https://github.com/restic/restic/issues/1542
+https://github.com/restic/restic/pull/2308
+https://github.com/restic/restic/pull/3210
+https://github.com/restic/restic/pull/3300

changelog/0.13.0_2022-03-26/issue-2202

@@ -0,0 +1,15 @@
+Enhancement: Add upload checksum for Azure, GS, S3 and Swift backends
+
+Previously only the B2 and partially the Swift backends verified the integrity
+of uploaded (encrypted) files. The verification works by informing the backend
+about the expected hash of the uploaded file. The backend then verifies the
+upload and thereby rules out any data corruption during upload.
+
+We have now added upload checksums for the Azure, GS, S3 and Swift backends,
+which besides integrity checking for uploads also means that restic can now be
+used to store backups in S3 buckets which have Object Lock enabled.
+
+https://github.com/restic/restic/issues/2202
+https://github.com/restic/restic/issues/2700
+https://github.com/restic/restic/issues/3023
+https://github.com/restic/restic/pull/3246

changelog/0.13.0_2022-03-26/issue-233

@@ -0,0 +1,29 @@
+Enhancement: Support negative include/exclude patterns
+
+If a pattern starts with an exclamation mark and it matches a file that was
+previously matched by a regular pattern, the match is cancelled. Notably,
+this can be used with `--exclude-file` to cancel the exclusion of some files.
+
+It works similarly to `.gitignore`, with the same limitation; Once a directory
+is excluded, it is not possible to include files inside the directory.
+
+Example of use as an exclude pattern for the `backup` command:
+
+    $HOME/**/*
+    !$HOME/Documents
+    !$HOME/code
+    !$HOME/.emacs.d
+    !$HOME/games
+    # [...]
+    node_modules
+    *~
+    *.o
+    *.lo
+    *.pyc
+    # [...]
+    $HOME/code/linux/*
+    !$HOME/code/linux/.git
+    # [...]
+
+https://github.com/restic/restic/issues/233
+https://github.com/restic/restic/pull/2311

changelog/0.13.0_2022-03-26/issue-2345

@@ -0,0 +1,13 @@
+Bugfix: Make cache crash-resistant and usable by multiple concurrent processes
+
+The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state
+in the event of restic (or the OS) crashing. This is now less likely to occur
+as files are downloaded to a temporary location before being moved to their
+proper location.
+
+This also allows multiple concurrent restic processes to operate on a single
+repository without conflicts. Previously, concurrent operations could cause
+segfaults because the processes saw each other's partially downloaded files.
+
+https://github.com/restic/restic/issues/2345
+https://github.com/restic/restic/pull/2838

changelog/0.13.0_2022-03-26/issue-2388

@@ -0,0 +1,10 @@
+Enhancement: Add warning for S3 if partial credentials are provided
+
+Previously restic did not notify about incomplete credentials when using the
+S3 backend, instead just reporting access denied.
+
+Restic now checks that both the AWS key ID and secret environment variables are
+set before connecting to the remote server, and reports an error if not.
+
+https://github.com/restic/restic/issues/2388
+https://github.com/restic/restic/pull/3532

changelog/0.13.0_2022-03-26/issue-2452

@@ -0,0 +1,11 @@
+Bugfix: Improve error handling of repository locking
+
+Previously, when the lock refresh failed to delete the old lock file, it forgot
+about the newly created one. Instead it continued trying to delete the old
+(usually no longer existing) lock file and thus over time lots of lock files
+accumulated. This has now been fixed.
+
+https://github.com/restic/restic/issues/2452
+https://github.com/restic/restic/issues/2473
+https://github.com/restic/restic/issues/2562
+https://github.com/restic/restic/pull/3512

changelog/0.13.0_2022-03-26/issue-2508

@@ -0,0 +1,8 @@
+Enhancement: Support JSON output and quiet mode for the `diff` command
+
+The `diff` command now supports outputting machine-readable output in JSON
+format. To enable this, pass the `--json` option to the command. To only print
+the summary and suppress detailed output, pass the `--quiet` option.
+
+https://github.com/restic/restic/issues/2508
+https://github.com/restic/restic/pull/3592

changelog/0.13.0_2022-03-26/issue-2656

@@ -0,0 +1,8 @@
+Enhancement: Add flag to disable TLS verification for self-signed certificates
+
+There is now an `--insecure-tls` global option in restic, which disables TLS
+verification for self-signed certificates in order to support some development
+workflows.
+
+https://github.com/restic/restic/issues/2656
+https://github.com/restic/restic/pull/2657

changelog/0.13.0_2022-03-26/issue-2738

@@ -0,0 +1,8 @@
+Bugfix: Don't print progress for `backup --json --quiet`
+
+Unlike the text output, the `--json` output format still printed progress
+information even in `--quiet` mode. This has now been fixed by always
+disabling the progress output in quiet mode.
+
+https://github.com/restic/restic/issues/2738
+https://github.com/restic/restic/pull/3264

changelog/0.13.0_2022-03-26/issue-3003

@@ -0,0 +1,9 @@
+Enhancement: Atomic uploads for the SFTP backend
+
+The SFTP backend did not upload files atomically. An interrupted upload could
+leave an incomplete file behind which could prevent restic from accessing the
+repository. This has now been fixed and uploads in the SFTP backend are done
+atomically.
+
+https://github.com/restic/restic/issues/3003
+https://github.com/restic/restic/pull/3524

changelog/0.13.0_2022-03-26/issue-3127

@@ -0,0 +1,6 @@
+Enhancement: Add xattr (extended attributes) support for Solaris
+
+Restic now supports xattr for the Solaris operating system.
+
+https://github.com/restic/restic/issues/3127
+https://github.com/restic/restic/pull/3628

changelog/0.13.0_2022-03-26/issue-3382

@@ -0,0 +1,8 @@
+Bugfix: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+
+Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment
+variable, which caused problems in certain system/usage configurations. This
+has now been fixed.
+
+https://github.com/restic/restic/issues/3382
+https://github.com/restic/restic/pull/3474

changelog/0.13.0_2022-03-26/issue-3464

@@ -0,0 +1,10 @@
+Enhancement: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+
+Restic used to silently ignore the `--no-lock` option of the `forget` command.
+
+It now skips creation of lock file in case both `--dry-run` and `--no-lock`
+are specified. If `--no-lock` option is specified without `--dry-run`, restic
+prints a warning message to stderr.
+
+https://github.com/restic/restic/issues/3464
+https://github.com/restic/restic/pull/3623

changelog/0.13.0_2022-03-26/issue-3490

@@ -0,0 +1,8 @@
+Enhancement: Support random subset by size in `check --read-data-subset`
+
+The `--read-data-subset` option of the `check` command now supports a third way
+of specifying the subset to check, namely `nS` where `n` is a size in bytes with
+suffix `S` as k/K, m/M, g/G or t/T.
+
+https://github.com/restic/restic/issues/3490
+https://github.com/restic/restic/pull/3548

changelog/0.13.0_2022-03-26/issue-3518

@@ -0,0 +1,8 @@
+Bugfix: Make `copy` command honor `--no-lock` for source repository
+
+The `copy` command previously did not respect the `--no-lock` option for the
+source repository, causing failures with read-only storage backends. This has
+now been fixed such that the option is now respected.
+
+https://github.com/restic/restic/issues/3518
+https://github.com/restic/restic/pull/3589

changelog/0.13.0_2022-03-26/issue-3541

@@ -0,0 +1,9 @@
+Enhancement: Improve handling of temporary B2 delete errors
+
+Deleting files on B2 could sometimes fail temporarily, which required restic to
+retry the delete operation. In some cases the file was deleted nevertheless,
+causing the retries and ultimately the restic command to fail. This has now been
+fixed.
+
+https://github.com/restic/restic/issues/3541
+https://github.com/restic/restic/pull/3544

changelog/0.13.0_2022-03-26/issue-3542

@@ -0,0 +1,8 @@
+Enhancement: Add file mode in symbolic notation to `ls --json`
+
+The `ls --json` command now provides the file mode in symbolic notation (using
+the `permissions` key), aligned with `find --json`.
+
+https://github.com/restic/restic/issues/3542
+https://github.com/restic/restic/pull/3573
+https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371

changelog/0.13.0_2022-03-26/issue-3556

@@ -0,0 +1,10 @@
+Bugfix: Fix hang with Backblaze B2 on SSL certificate authority error
+
+Previously, if a request failed with an SSL unknown certificate authority
+error, the B2 backend retried indefinitely and restic would appear to hang.
+
+This has now been fixed and restic instead fails with an error message.
+
+https://github.com/restic/restic/issues/3556
+https://github.com/restic/restic/issues/2355
+https://github.com/restic/restic/pull/3571

changelog/0.13.0_2022-03-26/issue-3601

@@ -0,0 +1,15 @@
+Bugfix: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+
+Previously, pressing Ctrl+C in a Windows console where restic was running with
+rclone as the backend would cause rclone to exit prematurely due to getting a
+`SIGINT` signal at the same time as restic. Restic would then wait for a long
+time for time with "unexpected EOF" and "rclone stdio connection already closed"
+errors.
+
+This has now been fixed by restic starting the rclone process detached from the
+console restic runs in (similar to starting processes in a new process group on
+Linux), which enables restic to gracefully clean up rclone (which now never gets
+the `SIGINT`).
+
+https://github.com/restic/restic/issues/3601
+https://github.com/restic/restic/pull/3602

changelog/0.13.0_2022-03-26/issue-3641

@@ -0,0 +1,14 @@
+Change: Ignore parent snapshot for `backup --stdin`
+
+Restic uses a parent snapshot to speed up directory scanning when performing
+backups, but this only wasted time and memory when the backup source is stdin
+(using the `--stdin` option of the `backup` command), since no directory scanning
+is performed in this case.
+
+Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows
+restic to skip some startup operations and saves a bit of resources.
+
+The `--parent` option is still available for `backup --stdin`, but is now ignored.
+
+https://github.com/restic/restic/issues/3641
+https://github.com/restic/restic/pull/3645

changelog/0.13.0_2022-03-26/issue-3667

@@ -0,0 +1,8 @@
+Bugfix: The `mount` command now reports symlinks sizes
+
+Symlinks used to have size zero in restic mountpoints, confusing some
+third-party tools. They now have a size equal to the byte length of their
+target path, as required by POSIX.
+
+https://github.com/restic/restic/issues/3667
+https://github.com/restic/restic/pull/3668

changelog/0.13.0_2022-03-26/pull-2594

@@ -0,0 +1,7 @@
+Enhancement: Speed up the `restore --verify` command
+
+The `--verify` option lets the `restore` command verify the file content
+after it has restored a snapshot. The performance of this operation has
+now been improved by up to a factor of two.
+
+https://github.com/restic/restic/pull/2594

changelog/0.13.0_2022-03-26/pull-2816

@@ -0,0 +1,10 @@
+Enhancement: The `backup` command no longer updates file access times on Linux
+
+When reading files during backup, restic used to cause the operating system to
+update the files' access times. Note that this did not apply to filesystems with
+disabled file access times.
+
+Restic now instructs the operating system not to update the file access time,
+if the user running restic is the file owner or has root permissions.
+
+https://github.com/restic/restic/pull/2816

changelog/0.13.0_2022-03-26/pull-2880

@@ -0,0 +1,9 @@
+Enhancement: Make `recover` collect only unreferenced trees
+
+Previously, the `recover` command used to generate a snapshot containing *all*
+root trees, even those which were already referenced by a snapshot.
+
+This has been improved such that it now only processes trees not already
+referenced by any snapshot.
+
+https://github.com/restic/restic/pull/2880

changelog/0.13.0_2022-03-26/pull-3429

@@ -0,0 +1,12 @@
+Enhancement: Verify that new or modified keys are stored correctly
+
+When adding a new key or changing the password of a key, restic used to just
+create the new key (and remove the old one, when changing the password). There
+was no verification that the new key was stored correctly and works properly.
+As the repository cannot be decrypted without a valid key file, this could in
+rare cases cause the repository to become inaccessible.
+
+Restic now checks that new key files actually work before continuing. This
+can protect against some (rare) cases of hardware or storage problems.
+
+https://github.com/restic/restic/pull/3429

changelog/0.13.0_2022-03-26/pull-3436

@@ -0,0 +1,12 @@
+Enhancement: Improve local backend's resilience to (system) crashes
+
+Restic now ensures that files stored using the `local` backend are created
+atomically (that is, files are either stored completely or not at all). This
+ensures that no incomplete files are left behind even if restic is terminated
+while writing a file.
+
+In addition, restic now tries to ensure that the directory in the repository
+which contains a newly uploaded file is also written to disk. This can prevent
+missing files if the system crashes or the disk is not properly unmounted.
+
+https://github.com/restic/restic/pull/3436

changelog/0.13.0_2022-03-26/pull-3488

@@ -0,0 +1,9 @@
+Bugfix: `rebuild-index` failed if an index file was damaged
+
+Previously, the `rebuild-index` command would fail with an error if an index
+file was damaged or truncated. This has now been fixed.
+
+On older restic versions, a (slow) workaround is to use
+`rebuild-index --read-all-packs` or to manually delete the damaged index.
+
+https://github.com/restic/restic/pull/3488

changelog/0.13.0_2022-03-26/pull-3508

@@ -0,0 +1,10 @@
+Enhancement: Cache blobs read by the `dump` command
+
+When dumping a file using the `dump` command, restic did not cache blobs in any
+way, so even consecutive runs of the same blob were loaded from the repository
+again and again, slowing down the dump.
+
+Now, the caching mechanism already used by the `fuse` command is also used by
+the `dump` command. This makes dumping much faster, especially for sparse files.
+
+https://github.com/restic/restic/pull/3508

changelog/0.13.0_2022-03-26/pull-3514

@@ -0,0 +1,8 @@
+Enhancement: Support configurable timeout for the rclone backend
+
+A slow rclone backend could cause restic to time out while waiting for the
+repository to open. Restic now offers an `-o rclone.timeout` option to make
+this timeout configurable.
+
+https://github.com/restic/restic/issues/3511
+https://github.com/restic/restic/pull/3514

changelog/0.13.0_2022-03-26/pull-3519

@@ -0,0 +1,6 @@
+Change: Require Go 1.14 or newer
+
+Restic now requires Go 1.14 to build. This allows it to use new
+standard library features instead of an external dependency.
+
+https://github.com/restic/restic/issues/3519

changelog/0.13.0_2022-03-26/pull-3591

@@ -0,0 +1,8 @@
+Bugfix: Fix handling of `prune --max-repack-size=0`
+
+Restic ignored the `--max-repack-size` option when passing a value of 0. This
+has now been fixed.
+
+As a workaround, `--max-repack-size=1` can be used with older versions of restic.
+
+https://github.com/restic/restic/pull/3591

changelog/0.13.0_2022-03-26/pull-3593

@@ -0,0 +1,10 @@
+Enhancement: Improve `copy` performance by parallelizing IO
+
+Restic copy previously only used a single thread for copying blobs between
+repositories, which resulted in limited performance when copying small blobs
+to/from a high latency backend (i.e. any remote backend, especially b2).
+
+Copying will now use 8 parallel threads to increase the throughput of the copy
+operation.
+
+https://github.com/restic/restic/pull/3593

changelog/0.13.0_2022-03-26/pull-3619

@@ -0,0 +1,11 @@
+Bugfix: Avoid choosing parent snapshots newer than time of new snapshot
+
+The `backup` command, when a `--parent` was not provided, previously chose the
+most recent matching snapshot as the parent snapshot. However, this didn't make
+sense when the user passed `--time` to create a new snapshot older than the most
+recent snapshot.
+
+Instead, `backup` now chooses the most recent snapshot which is not newer than
+the snapshot-being-created's timestamp, to avoid any time travel.
+
+https://github.com/restic/restic/pull/3619

changelog/0.14.0_2022-08-25/issue-1153

@@ -0,0 +1,9 @@
+Enhancement: Support pruning even when the disk is full
+
+When running out of disk space it was no longer possible to add or remove
+data from a repository. To help with recovering from such a deadlock, the
+prune command now supports an `--unsafe-recover-no-free-space` option to
+recover from these situations. Make sure to read the documentation first!
+
+https://github.com/restic/restic/issues/1153
+https://github.com/restic/restic/pull/3481

changelog/0.14.0_2022-08-25/issue-1842

@@ -0,0 +1,8 @@
+Change: Support debug log creation in release builds
+
+Creating a debug log was only possible in debug builds which required users to
+manually build restic. We changed the release builds to allow creating debug
+logs by simply setting the environment variable `DEBUG_LOG=logname.log`.
+
+https://github.com/restic/restic/issues/1842
+https://github.com/restic/restic/pull/3826

changelog/0.14.0_2022-08-25/issue-21

@@ -0,0 +1,28 @@
+Enhancement: Add compression support
+
+We've added compression support to the restic repository format. To create a
+repository using the new format run `init --repository-version 2`. Please note
+that the repository cannot be read by restic versions prior to 0.14.0.
+
+You can configure whether data is compressed with the option `--compression`. It
+can be set to `auto` (the default, which will compress very fast), `max` (which
+will trade backup speed and CPU usage for better compression), or `off` (which
+disables compression). Each setting is only applied for the current run of restic
+and does *not* apply to future runs. The option can also be set via the
+environment variable `RESTIC_COMPRESSION`.
+
+To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the
+documentation for more details. The migration checks the repository integrity
+and upgrades the repository format, but will not change any data. Afterwards,
+prune will rewrite the metadata to make use of compression.
+
+As an alternative you can use the `copy` command to migrate snapshots; First
+create a new repository using
+`init --repository-version 2 --copy-chunker-params --repo2 path/to/old/repo`,
+and then use the `copy` command to copy all snapshots to the new repository.
+
+https://github.com/restic/restic/issues/21
+https://github.com/restic/restic/issues/3779
+https://github.com/restic/restic/pull/3666
+https://github.com/restic/restic/pull/3704
+https://github.com/restic/restic/pull/3733

changelog/0.14.0_2022-08-25/issue-2162

@@ -0,0 +1,18 @@
+Enhancement: Adaptive IO concurrency based on backend connections
+
+Many commands used hard-coded limits for the number of concurrent operations.
+This prevented speed improvements by increasing the number of connections used
+by a backend.
+
+These limits have now been replaced by using the configured number of backend
+connections instead, which can be controlled using the
+`-o <backend-name>.connections=5` option. Commands will then automatically
+scale their parallelism accordingly.
+
+To limit the number of CPU cores used by restic, you can set the environment
+variable `GOMAXPROCS` accordingly. For example to use a single CPU core, use
+`GOMAXPROCS=1`.
+
+https://github.com/restic/restic/issues/2162
+https://github.com/restic/restic/issues/1467
+https://github.com/restic/restic/pull/3611

changelog/0.14.0_2022-08-25/issue-2248

@@ -0,0 +1,8 @@
+Bugfix: Support `self-update` on Windows
+
+Restic `self-update` would fail in situations where the operating system
+locks running binaries, including Windows. The new behavior works around
+this by renaming the running file and swapping the updated file in place.
+
+https://github.com/restic/restic/issues/2248
+https://github.com/restic/restic/pull/3675

changelog/0.14.0_2022-08-25/issue-2291

@@ -0,0 +1,12 @@
+Enhancement: Allow pack size customization
+
+Restic now uses a target pack size of 16 MiB by default. This can be customized
+using the `--pack-size size` option. Supported pack sizes range between 4 and
+128 MiB.
+
+It is possible to migrate an existing repository to _larger_ pack files using
+`prune --repack-small`. This will rewrite every pack file which is
+significantly smaller than the target size.
+
+https://github.com/restic/restic/issues/2291
+https://github.com/restic/restic/pull/3731

changelog/0.14.0_2022-08-25/issue-2295

@@ -0,0 +1,14 @@
+Enhancement: Allow use of SAS token to authenticate to Azure
+
+Previously restic only supported AccountKeys to authenticate to Azure
+storage accounts, which necessitates giving a significant amount of
+access.
+
+We added support for Azure SAS tokens which are a more fine-grained
+and time-limited manner of granting access. Set the `AZURE_ACCOUNT_NAME`
+and `AZURE_ACCOUNT_SAS` environment variables to use a SAS token for
+authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it will take
+precedence.
+
+https://github.com/restic/restic/issues/2295
+https://github.com/restic/restic/pull/3661

changelog/0.14.0_2022-08-25/issue-2696

@@ -0,0 +1,13 @@
+Enhancement: Improve backup speed with many small files
+
+We have restructured the backup pipeline to continue reading files while all
+upload connections are busy. This allows the backup to already prepare the next
+data file such that the upload can continue as soon as a connection becomes
+available. This can especially improve the backup performance for high latency
+backends.
+
+The upload concurrency is now controlled using the `-o <backend-name>.connections=5`
+option.
+
+https://github.com/restic/restic/issues/2696
+https://github.com/restic/restic/pull/3489

changelog/0.14.0_2022-08-25/issue-2907

@@ -0,0 +1,15 @@
+Enhancement: Make snapshot directory structure of `mount` command customizable
+
+We've added the possibility to customize the snapshot directory structure of
+the `mount` command using templates passed to the `--snapshot-template` option.
+The formatting of snapshots' timestamps is now controlled using `--time-template`
+and supports subdirectories to for example group snapshots by year. Please
+see `restic help mount` for further details.
+
+Characters in tag names which are not allowed in a filename are replaced by
+underscores `_`. For example a tag `foo/bar` will result in a directory name
+of `foo_bar`.
+
+https://github.com/restic/restic/issues/2907
+https://github.com/restic/restic/pull/2913
+https://github.com/restic/restic/pull/3691

changelog/0.14.0_2022-08-25/issue-3114

@@ -0,0 +1,12 @@
+Enhancement: Optimize handling of duplicate blobs in `prune`
+
+Restic `prune` always used to repack all data files containing duplicate
+blobs. This effectively removed all duplicates during prune. However, as a
+consequence all these data files were repacked even if the unused repository
+space threshold could be reached with less work.
+
+This is now changed and `prune` works nice and fast even when there are lots
+of duplicate blobs.
+
+https://github.com/restic/restic/issues/3114
+https://github.com/restic/restic/pull/3290

changelog/0.14.0_2022-08-25/issue-3295

@@ -0,0 +1,13 @@
+Change: Deprecate `check --check-unused` and add further checks
+
+Since restic 0.12.0, it is expected to still have unused blobs after running
+`prune`. This made the `--check-unused` option of the `check` command rather
+useless and tended to confuse users. This option has been deprecated and is
+now ignored.
+
+The `check` command now also warns if a repository is using either the legacy
+S3 layout or mixed pack files with both tree and data blobs. The latter is
+known to cause performance problems.
+
+https://github.com/restic/restic/issues/3295
+https://github.com/restic/restic/pull/3730

changelog/0.14.0_2022-08-25/issue-3428

@@ -0,0 +1,14 @@
+Bugfix: List snapshots in backend at most once to resolve snapshot IDs
+
+Many commands support specifying a list of snapshot IDs which are then used to
+determine the snapshots to be processed by the command. To resolve snapshot IDs
+or `latest`, and check that these exist, restic previously listed all snapshots
+stored in the repository. Depending on the backend this could be a slow and/or
+expensive operation.
+
+Restic now lists the snapshots only once and remembers the result in order to
+resolve all further snapshot IDs swiftly.
+
+https://github.com/restic/restic/issues/3428
+https://github.com/restic/restic/pull/3570
+https://github.com/restic/restic/pull/3395

changelog/0.14.0_2022-08-25/issue-3432

@@ -0,0 +1,14 @@
+Bugfix: Fix rare 'not found in repository' error for `copy` command
+
+In rare cases `copy` (and other commands) would report that `LoadTree(...)`
+returned an `id [...] not found in repository` error. This could be caused by
+a backup or copy command running concurrently. The error was only temporary;
+running the failed restic command a second time as a workaround did resolve the
+error.
+
+This issue has now been fixed by correcting the order in which restic reads data
+from the repository. It is now guaranteed that restic only loads snapshots for
+which all necessary data is already available.
+
+https://github.com/restic/restic/issues/3432
+https://github.com/restic/restic/pull/3570

changelog/0.14.0_2022-08-25/issue-3465

@@ -0,0 +1,10 @@
+Enhancement: Improve handling of temporary files on Windows
+
+In some cases restic failed to delete temporary files, causing the current
+command to fail. This has now been fixed by ensuring that Windows automatically
+deletes the file. In addition, temporary files are only written to disk when
+necessary, reducing disk writes.
+
+https://github.com/restic/restic/issues/3465
+https://github.com/restic/restic/issues/1551
+https://github.com/restic/restic/pull/3610

changelog/0.14.0_2022-08-25/issue-3685

@@ -0,0 +1,7 @@
+Bugfix: The `diff` command incorrectly listed some files as added
+
+There was a bug in the `diff` command, causing it to always show files in a
+removed directory as added. This has now been fixed.
+
+https://github.com/restic/restic/issues/3685
+https://github.com/restic/restic/pull/3686

changelog/0.14.0_2022-08-25/issue-3692

@@ -0,0 +1,13 @@
+Bugfix: Fix rclone (shimmed by Scoop) and sftp not working on Windows
+
+In #3602 a fix was introduced to address the problem of `rclone` prematurely
+exiting when Ctrl+C is pressed on Windows. The solution was to create the
+subprocess with its console detached from the restic console.
+
+However, this solution failed when using `rclone` installed by Scoop or using
+`sftp` with a passphrase-protected private key. We've now fixed this by using
+a different approach to prevent Ctrl-C from passing down too early.
+
+https://github.com/restic/restic/issues/3681
+https://github.com/restic/restic/issues/3692
+https://github.com/restic/restic/pull/3696

changelog/0.14.0_2022-08-25/issue-3709

@@ -0,0 +1,11 @@
+Enhancement: Validate exclude patterns before backing up
+
+Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or
+`--iexclude-file` previously weren't validated. As a consequence, invalid
+patterns resulted in files that were meant to be excluded being backed up.
+
+Restic now validates all patterns before running the backup and aborts with
+a fatal error if an invalid pattern is detected.
+
+https://github.com/restic/restic/issues/3709
+https://github.com/restic/restic/pull/3734

changelog/0.14.0_2022-08-25/issue-3720

@@ -0,0 +1,13 @@
+Bugfix: Directory sync errors for repositories accessed via SMB
+
+On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in
+restic failing to save the lock file, yielding the following errors:
+
+Save(<lock/071fe833f0>) returned error, retrying after 552.330144ms: sync /repo/locks: no such file or directory
+Save(<lock/bf789d7343>) returned error, retrying after 552.330144ms: sync /repo/locks: invalid argument
+
+This has now been fixed by ignoring the relevant error codes.
+
+https://github.com/restic/restic/issues/3720
+https://github.com/restic/restic/issues/3751
+https://github.com/restic/restic/pull/3752

changelog/0.14.0_2022-08-25/issue-3736

@@ -0,0 +1,8 @@
+Bugfix: The `stats` command miscalculated restore size for multiple snapshots
+
+Since restic 0.10.0 the restore size calculated by the `stats` command for
+multiple snapshots was too low. The hardlink detection was accidentally applied
+across multiple snapshots and thus ignored many files. This has now been fixed.
+
+https://github.com/restic/restic/issues/3736
+https://github.com/restic/restic/pull/3740

changelog/0.14.0_2022-08-25/issue-3837

@@ -0,0 +1,8 @@
+Enhancement: Improve SFTP repository initialization over slow links
+
+The `init` command, when used on an SFTP backend, now sends multiple `mkdir`
+commands to the backend concurrently. This reduces the waiting times when
+creating a repository over a very slow connection.
+
+https://github.com/restic/restic/issues/3837
+https://github.com/restic/restic/pull/3840

changelog/0.14.0_2022-08-25/issue-3861

@@ -0,0 +1,9 @@
+Bugfix: Yield error on invalid policy to `forget`
+
+The `forget` command previously silently ignored invalid/unsupported
+units in the duration options, such as e.g. `--keep-within-daily 2w`.
+
+Specifying an invalid/unsupported duration unit now results in an error.
+
+https://github.com/restic/restic/issues/3861
+https://github.com/restic/restic/pull/3862

changelog/0.14.0_2022-08-25/pull-3419

@@ -0,0 +1,21 @@
+Enhancement: Use config file permissions to control file group access
+
+Previously files in a local/SFTP repository would always end up with very
+restrictive access permissions, allowing access only to the owner. This
+prevented a number of valid use-cases involving groups and ACLs.
+
+We now use the permissions of the config file in the repository to decide
+whether group access should be given to newly created repository files or
+not. We arrange for repository files to be created group readable exactly
+when the repository config file is group readable.
+
+To opt-in to group readable repositories, a simple `chmod -R g+r` or
+equivalent on the config file can be used. For repositories that should
+be writable by group members a tad more setup is required, see the docs.
+
+Posix ACLs can also be used now that the group permissions being forced to
+zero no longer masks the effect of ACL entries.
+
+https://github.com/restic/restic/issues/2351
+https://github.com/restic/restic/pull/3419
+https://forum.restic.net/t/1391

changelog/0.14.0_2022-08-25/pull-3475

@@ -0,0 +1,9 @@
+Enhancement: Allow limiting IO concurrency for local and SFTP backend
+
+Restic did not support limiting the IO concurrency / number of connections for
+accessing repositories stored using the local or SFTP backends. The number of
+connections is now limited as for other backends, and can be configured via the
+the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures
+that restic does not overwhelm the backend with concurrent IO operations.
+
+https://github.com/restic/restic/pull/3475

changelog/0.14.0_2022-08-25/pull-3484

@@ -0,0 +1,13 @@
+Enhancement: Stream data in `check` and `prune` commands
+
+The commands `check --read-data` and `prune` previously downloaded data files
+into temporary files which could end up being written to disk. This could cause
+a large amount of data being written to disk.
+
+The pack files are now instead streamed, which removes the need for temporary
+files. Please note that *uploads* during `backup` and `prune` still require
+temporary files.
+
+https://github.com/restic/restic/pull/3484
+https://github.com/restic/restic/issues/3710
+https://github.com/restic/restic/pull/3717

changelog/0.14.0_2022-08-25/pull-3513

@@ -0,0 +1,10 @@
+Enhancement: Improve speed of `copy` command
+
+The `copy` command could require a long time to copy snapshots for non-local
+backends. This has been improved to provide a throughput comparable to the
+`restore` command.
+
+Additionally, `copy` now displays a progress bar.
+
+https://github.com/restic/restic/issues/2923
+https://github.com/restic/restic/pull/3513

changelog/0.14.0_2022-08-25/pull-3680

@@ -0,0 +1,8 @@
+Change: Update dependencies and require Go 1.15 or newer
+
+We've updated most dependencies. Since some libraries require newer language
+features we're dropping support for Go 1.14, which means that restic now
+requires at least Go 1.15 to build.
+
+https://github.com/restic/restic/issues/3680
+https://github.com/restic/restic/issues/3883

changelog/0.14.0_2022-08-25/pull-3716

@@ -0,0 +1,7 @@
+Bugfix: Print "wrong password" to stderr instead of stdout
+
+If an invalid password was entered, the error message was printed on stdout and
+not on stderr as intended. This has now been fixed.
+
+https://github.com/restic/restic/pull/3716
+https://forum.restic.net/t/4965

changelog/0.14.0_2022-08-25/pull-3729

@@ -0,0 +1,8 @@
+Enhancement: Display full IDs in `check` warnings
+
+When running commands to inspect or repair a damaged repository, it is often
+necessary to supply the full IDs of objects stored in the repository.
+
+The output of `check` now includes full IDs instead of their shortened variant.
+
+https://github.com/restic/restic/pull/3729

changelog/0.14.0_2022-08-25/pull-3742

@@ -0,0 +1,19 @@
+Change: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
+
+The `init` and `copy` commands can read data from another repository.
+However, confusingly `--repo2` referred to the repository *from* which the
+`init` command copies parameters, but for the `copy` command `--repo2`
+referred to the copy *destination*.
+
+We've introduced a new option, `--from-repo`, which always refers to the
+source repository for both commands. The old parameter names have been
+deprecated but still work. To create a new repository and copy all snapshots
+to it, the commands are now as follows:
+
+```
+restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params
+restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo
+```
+
+https://github.com/restic/restic/pull/3742
+https://forum.restic.net/t/5017

changelog/0.14.0_2022-08-25/pull-3772

@@ -0,0 +1,14 @@
+Bugfix: Correctly rebuild index for legacy repositories
+
+After running `rebuild-index` on a legacy repository containing mixed pack
+files (that is, pack files which store both metadata and file data), `check`
+printed warnings like `pack 12345678 contained in several indexes: ...`.
+This warning was not critical, but has now nonetheless been fixed by properly
+handling mixed pack files while rebuilding the index.
+
+Running `prune` for such legacy repositories will also fix the warning by
+reorganizing the pack files which caused it.
+
+https://github.com/restic/restic/pull/3772
+https://github.com/restic/restic/pull/3884
+https://forum.restic.net/t/5044/13

changelog/0.14.0_2022-08-25/pull-3773

@@ -0,0 +1,7 @@
+Enhancement: Optimize memory usage for directories with many files
+
+Backing up a directory with hundreds of thousands or more files caused restic
+to require large amounts of memory. We've now optimized the `backup` command
+such that it requires up to 30% less memory.
+
+https://github.com/restic/restic/pull/3773

changelog/0.14.0_2022-08-25/pull-3776

@@ -0,0 +1,11 @@
+Bugfix: Limit number of key files tested while opening a repository
+
+Previously, restic tested the password against every key in the repository
+when opening a repository. The more keys there were in the repository, the
+slower this operation became.
+
+Restic now tests the password against up to 20 key files in the repository.
+Alternatively, you can use the `--key-hint=<key ID>` option to specify a
+specific key file to use instead.
+
+https://github.com/restic/restic/pull/3776

changelog/0.14.0_2022-08-25/pull-3819

@@ -0,0 +1,11 @@
+Enhancement: Validate include/exclude patterns before restoring
+
+Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include`
+and `--iinclude` weren't validated before running the restore. Invalid
+patterns would result in error messages being printed repeatedly, and
+possibly unwanted files being restored.
+
+Restic now validates all patterns before running the restore, and aborts
+with a fatal error if an invalid pattern is detected.
+
+https://github.com/restic/restic/pull/3819

changelog/0.7.1_2017-07-22/issue-1081

@@ -1,4 +1,4 @@
-Enhancement: Clarify semantic for `--tasg` for the `forget` command
+Enhancement: Clarify semantic for `--tag` for the `forget` command
 
 https://github.com/restic/restic/issues/1081
 https://github.com/restic/restic/pull/1090

changelog/TEMPLATE

@@ -1,5 +1,5 @@
 # The first line must start with Bugfix:, Enhancement: or Change:,
-# including the colon. Use present use. Remove lines starting with '#'
+# including the colon. Use present tense. Remove lines starting with '#'
 # from this template.
 Enhancement: Allow custom bar in the foo command
 

cmd/restic/cleanup.go

@@ -58,7 +58,7 @@ func RunCleanupHandlers() {
 func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
-		Warnf("%ssignal %v received, cleaning up\n", ClearLine(), s)
+		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
 
 		code := 0
 

cmd/restic/cmd_backup.go

@@ -12,25 +12,26 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
-	tomb "gopkg.in/tomb.v2"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui"
-	"github.com/restic/restic/internal/ui/json"
+	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 var cmdBackup = &cobra.Command{
-	Use:   "backup [flags] FILE/DIR [FILE/DIR] ...",
+	Use:   "backup [flags] [FILE/DIR] ...",
 	Short: "Create a new backup of files and/or directories",
 	Long: `
 The "backup" command creates a new snapshot and saves the files and directories
@@ -55,16 +56,22 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		var t tomb.Tomb
-		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
-		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
+		var wg sync.WaitGroup
+		cancelCtx, cancel := context.WithCancel(globalOptions.ctx)
+		defer func() {
+			// shutdown termstatus
+			cancel()
+			wg.Wait()
+		}()
 
-		err := runBackup(backupOptions, globalOptions, term, args)
-		t.Kill(nil)
-		if werr := t.Wait(); werr != nil {
-			panic(fmt.Sprintf("term.Run() returned err: %v", err))
-		}
-		return err
+		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			term.Run(cancelCtx)
+		}()
+
+		return runBackup(backupOptions, globalOptions, term, args)
 	},
 }
 
@@ -92,6 +99,7 @@ type BackupOptions struct {
 	IgnoreInode             bool
 	IgnoreCtime             bool
 	UseFsSnapshot           bool
+	DryRun                  bool
 }
 
 var backupOptions BackupOptions
@@ -103,7 +111,7 @@ func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
@@ -132,6 +140,7 @@ func init() {
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
@@ -142,7 +151,7 @@ func init() {
 func filterExisting(items []string) (result []string, err error) {
 	for _, item := range items {
 		_, err := fs.Lstat(item)
-		if err != nil && os.IsNotExist(errors.Cause(err)) {
+		if errors.Is(err, os.ErrNotExist) {
 			Warnf("%v does not exist, skipping\n", item)
 			continue
 		}
@@ -297,6 +306,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 		if err != nil {
 			return nil, err
 		}
+
+		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
+			return nil, errors.Fatalf("--exclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		opts.Excludes = append(opts.Excludes, excludes...)
 	}
 
@@ -305,14 +319,27 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 		if err != nil {
 			return nil, err
 		}
+
+		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
+			return nil, errors.Fatalf("--iexclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
 	}
 
 	if len(opts.InsensitiveExcludes) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExcludes); !valid {
+			return nil, errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
 	}
 
 	if len(opts.Excludes) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.Excludes); !valid {
+			return nil, errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		fs = append(fs, rejectByPattern(opts.Excludes))
 	}
 
@@ -471,10 +498,10 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
-		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
+		id, err := restic.FindSnapshot(ctx, repo.Backend(), opts.Parent)
 		if err != nil {
 			return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
 		}
@@ -484,7 +511,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host})
+		id, err := restic.FindLatestSnapshot(ctx, repo.Backend(), repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -514,8 +541,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 	}
 
-	var t tomb.Tomb
-
 	if gopts.verbosity >= 2 && !gopts.JSON {
 		Verbosef("open repository\n")
 	}
@@ -525,33 +550,17 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	type ArchiveProgressReporter interface {
-		CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration)
-		StartFile(filename string)
-		CompleteBlob(filename string, bytes uint64)
-		ScannerError(item string, fi os.FileInfo, err error) error
-		ReportTotal(item string, s archiver.ScanStats)
-		SetMinUpdatePause(d time.Duration)
-		Run(ctx context.Context) error
-		Error(item string, fi os.FileInfo, err error) error
-		Finish(snapshotID restic.ID)
-
-		// ui.StdioWrapper
-		Stdout() io.WriteCloser
-		Stderr() io.WriteCloser
-
-		// ui.Message
-		E(msg string, args ...interface{})
-		P(msg string, args ...interface{})
-		V(msg string, args ...interface{})
-		VV(msg string, args ...interface{})
-	}
-
-	var p ArchiveProgressReporter
+	var progressPrinter backup.ProgressPrinter
 	if gopts.JSON {
-		p = json.NewBackup(term, gopts.verbosity)
+		progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
 	} else {
-		p = ui.NewBackup(term, gopts.verbosity)
+		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
+	}
+	progressReporter := backup.NewProgress(progressPrinter)
+
+	if opts.DryRun {
+		repo.SetDryRun()
+		progressReporter.SetDryRun()
 	}
 
 	// use the terminal for stdout/stderr
@@ -559,14 +568,17 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	defer func() {
 		gopts.stdout, gopts.stderr = prevStdout, prevStderr
 	}()
-	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
+	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
 
-	p.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet))
+	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 
-	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
+	wg, wgCtx := errgroup.WithContext(gopts.ctx)
+	cancelCtx, cancel := context.WithCancel(wgCtx)
+	defer cancel()
+	wg.Go(func() error { return progressReporter.Run(cancelCtx) })
 
 	if !gopts.JSON {
-		p.V("lock repository")
+		progressPrinter.V("lock repository")
 	}
 	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
@@ -586,27 +598,30 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
+	var parentSnapshotID *restic.ID
+	if !opts.Stdin {
+		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
+		if err != nil {
+			return err
+		}
+
+		if !gopts.JSON {
+			if parentSnapshotID != nil {
+				progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
+			} else {
+				progressPrinter.P("no parent snapshot found, will read all files\n")
+			}
+		}
+	}
+
 	if !gopts.JSON {
-		p.V("load index files")
+		progressPrinter.V("load index files")
 	}
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 
-	parentSnapshotID, err := findParentSnapshot(gopts.ctx, repo, opts, targets)
-	if err != nil {
-		return err
-	}
-
-	if !gopts.JSON {
-		if parentSnapshotID != nil {
-			p.P("using parent snapshot %v\n", parentSnapshotID.Str())
-		} else {
-			p.P("no parent snapshot found, will read all files\n")
-		}
-	}
-
 	selectByNameFilter := func(item string) bool {
 		for _, reject := range rejectByNameFuncs {
 			if reject(item) {
@@ -632,12 +647,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 
 		errorHandler := func(item string, err error) error {
-			return p.Error(item, nil, err)
+			return progressReporter.Error(item, err)
 		}
 
 		messageHandler := func(msg string, args ...interface{}) {
 			if !gopts.JSON {
-				p.P(msg, args...)
+				progressPrinter.P(msg, args...)
 			}
 		}
 
@@ -647,7 +662,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 	if opts.Stdin {
 		if !gopts.JSON {
-			p.V("read data from stdin")
+			progressPrinter.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
 		targetFS = &fs.Reader{
@@ -662,26 +677,26 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	sc := archiver.NewScanner(targetFS)
 	sc.SelectByName = selectByNameFilter
 	sc.Select = selectFilter
-	sc.Error = p.ScannerError
-	sc.Result = p.ReportTotal
+	sc.Error = progressReporter.ScannerError
+	sc.Result = progressReporter.ReportTotal
 
 	if !gopts.JSON {
-		p.V("start scan on %v", targets)
+		progressPrinter.V("start scan on %v", targets)
 	}
-	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
+	wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 
 	arch := archiver.New(repo, targetFS, archiver.Options{})
 	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
 	success := true
-	arch.Error = func(item string, fi os.FileInfo, err error) error {
+	arch.Error = func(item string, err error) error {
 		success = false
-		return p.Error(item, fi, err)
+		return progressReporter.Error(item, err)
 	}
-	arch.CompleteItem = p.CompleteItem
-	arch.StartFile = p.StartFile
-	arch.CompleteBlob = p.CompleteBlob
+	arch.CompleteItem = progressReporter.CompleteItem
+	arch.StartFile = progressReporter.StartFile
+	arch.CompleteBlob = progressReporter.CompleteBlob
 
 	if opts.IgnoreInode {
 		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
@@ -705,15 +720,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	if !gopts.JSON {
-		p.V("start backup on %v", targets)
+		progressPrinter.V("start backup on %v", targets)
 	}
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
 
 	// cleanly shutdown all running goroutines
-	t.Kill(nil)
+	cancel()
 
 	// let's see if one returned an error
-	werr := t.Wait()
+	werr := wg.Wait()
 
 	// return original error
 	if err != nil {
@@ -721,9 +736,9 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	// Report finished execution
-	p.Finish(id)
-	if !gopts.JSON {
-		p.P("snapshot %s saved\n", id.Str())
+	progressReporter.Finish(id)
+	if !gopts.JSON && !opts.DryRun {
+		progressPrinter.P("snapshot %s saved\n", id.Str())
 	}
 	if !success {
 		return ErrInvalidSourceData

cmd/restic/cmd_cache.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/cache"
@@ -140,8 +141,13 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
 		}
 
+		name := entry.Name()
+		if !strings.HasPrefix(name, "restic-check-cache-") {
+			name = name[:10]
+		}
+
 		tab.AddRow(data{
-			entry.Name()[:10],
+			name,
 			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
 			old,
 			size,

cmd/restic/cmd_cat.go

@@ -62,7 +62,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			}
 
 			// find snapshot id with prefix
-			id, err = restic.FindSnapshot(gopts.ctx, repo, args[1])
+			id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1])
 			if err != nil {
 				return errors.Fatalf("could not find snapshot: %v\n", err)
 			}
@@ -79,7 +79,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadAndDecrypt(gopts.ctx, nil, restic.IndexFile, id)
+		buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil)
 		if err != nil {
 			return err
 		}
@@ -87,13 +87,12 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn := &restic.Snapshot{}
-		err = repo.LoadJSONUnpacked(gopts.ctx, restic.SnapshotFile, id, sn)
+		sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
 		if err != nil {
 			return err
 		}
 
-		buf, err := json.MarshalIndent(&sn, "", "  ")
+		buf, err := json.MarshalIndent(sn, "", "  ")
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_check.go

@@ -5,10 +5,12 @@ import (
 	"math/rand"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
@@ -54,8 +56,14 @@ func init() {
 
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
-	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
+	var ignored bool
+	f.BoolVar(&ignored, "check-unused", false, "find unused blobs")
+	err := f.MarkDeprecated("check-unused", "`--check-unused` is deprecated and will be ignored")
+	if err != nil {
+		// MarkDeprecated only returns an error when the flag is not found
+		panic(err)
+	}
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
 
@@ -65,7 +73,7 @@ func checkFlags(opts CheckOptions) error {
 	}
 	if opts.ReadDataSubset != "" {
 		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
+		argumentError := errors.Fatal("check flag --read-data-subset has invalid value, please see documentation")
 		if err == nil {
 			if len(dataSubset) != 2 {
 				return argumentError
@@ -76,7 +84,7 @@ func checkFlags(opts CheckOptions) error {
 			if dataSubset[1] > totalBucketsMax {
 				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
 			}
-		} else {
+		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
 			percentage, err := parsePercentage(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
@@ -84,8 +92,19 @@ func checkFlags(opts CheckOptions) error {
 
 			if percentage <= 0.0 || percentage > 100.0 {
 				return errors.Fatal(
-					"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
+					"check flag --read-data-subset=x% x must be above 0.0% and at most 100.0%")
 			}
+
+		} else {
+			fileSize, err := parseSizeStr(opts.ReadDataSubset)
+			if err != nil {
+				return argumentError
+			}
+			if fileSize <= 0.0 {
+				return errors.Fatal(
+					"check flag --read-data-subset=n n must be above 0")
+			}
+
 		}
 	}
 
@@ -129,10 +148,10 @@ func parsePercentage(s string) (float64, error) {
 
 // prepareCheckCache configures a special cache directory for check.
 //
-//  * if --with-cache is specified, the default cache is used
-//  * if the user explicitly requested --no-cache, we don't use any cache
-//  * if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
-//  * by default, we use a cache in a temporary directory that is deleted after the check
+//   - if --with-cache is specified, the default cache is used
+//   - if the user explicitly requested --no-cache, we don't use any cache
+//   - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
+//   - by default, we use a cache in a temporary directory that is deleted after the check
 func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
 	cleanup = func() {}
 	if opts.WithCache {
@@ -146,6 +165,9 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	}
 
 	cachedir := gopts.CacheDir
+	if cachedir == "" {
+		cachedir = cache.EnvDir()
+	}
 
 	// use a cache in a temporary directory
 	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
@@ -195,21 +217,37 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	chkr := checker.New(repo, opts.CheckUnused)
+	err = chkr.LoadSnapshots(gopts.ctx)
+	if err != nil {
+		return err
+	}
 
 	Verbosef("load indexes\n")
 	hints, errs := chkr.LoadIndex(gopts.ctx)
 
-	dupFound := false
+	errorsFound := false
+	suggestIndexRebuild := false
+	mixedFound := false
 	for _, hint := range hints {
-		Printf("%v\n", hint)
-		if _, ok := hint.(checker.ErrDuplicatePacks); ok {
-			dupFound = true
+		switch hint.(type) {
+		case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
+			Printf("%v\n", hint)
+			suggestIndexRebuild = true
+		case *checker.ErrMixedPack:
+			Printf("%v\n", hint)
+			mixedFound = true
+		default:
+			Warnf("error: %v\n", hint)
+			errorsFound = true
 		}
 	}
 
-	if dupFound {
+	if suggestIndexRebuild {
 		Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
 	}
+	if mixedFound {
+		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
+	}
 
 	if len(errs) > 0 {
 		for _, err := range errs {
@@ -218,7 +256,6 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("LoadIndex returned errors")
 	}
 
-	errorsFound := false
 	orphanedPacks := 0
 	errChan := make(chan error)
 
@@ -229,19 +266,25 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		if checker.IsOrphanedPack(err) {
 			orphanedPacks++
 			Verbosef("%v\n", err)
-			continue
+		} else if _, ok := err.(*checker.ErrLegacyLayout); ok {
+			Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
+		} else {
+			errorsFound = true
+			Warnf("%v\n", err)
 		}
-		errorsFound = true
-		Warnf("%v\n", err)
 	}
 
 	if orphanedPacks > 0 {
-		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nYou can run `restic prune` to correct this.\n", orphanedPacks)
+		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
 	}
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
+	var wg sync.WaitGroup
+
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
 		chkr.Structure(gopts.ctx, bar, errChan)
@@ -249,7 +292,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	for err := range errChan {
 		errorsFound = true
-		if e, ok := err.(checker.TreeError); ok {
+		if e, ok := err.(*checker.TreeError); ok {
 			Warnf("error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
 				Warnf("  %v\n", treeErr)
@@ -259,6 +302,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	// Wait for the progress bar to be complete before printing more below.
+	// Must happen after `errChan` is read from in the above loop to avoid
+	// deadlocking in the case of errors.
+	wg.Wait()
+
 	if opts.CheckUnused {
 		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
 			Verbosef("unused blob %v\n", id)
@@ -294,10 +342,27 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
 			packCount := uint64(len(packs))
 			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
+		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
+			percentage, err := parsePercentage(opts.ReadDataSubset)
+			if err == nil {
+				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
+				Verbosef("read %.1f%% of data packs\n", percentage)
+			}
 		} else {
-			percentage, _ := parsePercentage(opts.ReadDataSubset)
-			packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
-			Verbosef("read %.1f%% of data packs\n", percentage)
+			repoSize := int64(0)
+			allPacks := chkr.GetPacks()
+			for _, size := range allPacks {
+				repoSize += size
+			}
+			if repoSize == 0 {
+				return errors.Fatal("Cannot read from a repository having size 0")
+			}
+			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
+			if subsetSize > repoSize {
+				subsetSize = repoSize
+			}
+			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
+			Verbosef("read %d bytes of data packs\n", subsetSize)
 		}
 		if packs == nil {
 			return errors.Fatal("internal error: failed to select packs to check")
@@ -349,6 +414,11 @@ func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage floa
 		id := keys[idx[i]]
 		packs[id] = allPacks[id]
 	}
-
+	return packs
+}
+
+func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, repoSize int64) map[restic.ID]int64 {
+	subsetPercentage := (float64(subsetSize) / float64(repoSize)) * 100.0
+	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }

cmd/restic/cmd_check_test.go

@@ -129,3 +129,37 @@ func TestSelectNoRandomPacksByPercentage(t *testing.T) {
 	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
 }
+
+func TestSelectRandomPacksByFileSize(t *testing.T) {
+	var testPacks = make(map[restic.ID]int64)
+	for i := 1; i <= 10; i++ {
+		id := restic.NewRandomID()
+		// ensure unique ids
+		id[0] = byte(i)
+		testPacks[id] = 0
+	}
+
+	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
+	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
+
+	selectedPacks = selectRandomPacksByFileSize(testPacks, 10240, 51200)
+	rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected selection")
+	}
+
+	selectedPacks = selectRandomPacksByFileSize(testPacks, 500, 500)
+	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected item in selection")
+	}
+}
+
+func TestSelectNoRandomPacksByFileSize(t *testing.T) {
+	// that the a repository without pack files works
+	var testPacks = make(map[restic.ID]int64)
+	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
+	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
+}

cmd/restic/cmd_copy.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
 
@@ -48,17 +50,21 @@ func init() {
 	cmdRoot.AddCommand(cmdCopy)
 
 	f := cmdCopy.Flags()
-	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots to")
+	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
 	f.StringArrayVarP(&copyOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
 	f.Var(&copyOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
 	f.StringArrayVar(&copyOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 }
 
 func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
-	dstGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
+	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
 	if err != nil {
 		return err
 	}
+	if isFromRepo {
+		// swap global options, if the secondary repo was set via from-repo
+		gopts, secondaryGopts = secondaryGopts, gopts
+	}
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
@@ -68,15 +74,17 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	dstRepo, err := OpenRepository(dstGopts)
+	dstRepo, err := OpenRepository(secondaryGopts)
 	if err != nil {
 		return err
 	}
 
-	srcLock, err := lockRepo(ctx, srcRepo)
-	defer unlockRepo(srcLock)
-	if err != nil {
-		return err
+	if !gopts.NoLock {
+		srcLock, err := lockRepo(ctx, srcRepo)
+		defer unlockRepo(srcLock)
+		if err != nil {
+			return err
+		}
 	}
 
 	dstLock, err := lockRepo(ctx, dstRepo)
@@ -85,6 +93,16 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
+	srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
+	dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	debug.Log("Loading source index")
 	if err := srcRepo.LoadIndex(ctx); err != nil {
 		return err
@@ -96,7 +114,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
-	for sn := range FindFilteredSnapshots(ctx, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
+	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
 		if sn.Original != nil && !sn.Original.IsNull() {
 			dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
 		}
@@ -107,7 +125,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 	// remember already processed trees across all snapshots
 	visitedTrees := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
 
 		// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
@@ -129,24 +147,18 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 			}
 		}
 		Verbosef("  copy started, this may take a while...\n")
-
-		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree); err != nil {
+		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
 			return err
 		}
 		debug.Log("tree copied")
 
-		if err = dstRepo.Flush(ctx); err != nil {
-			return err
-		}
-		debug.Log("flushed packs and saved index")
-
 		// save snapshot
 		sn.Parent = nil // Parent does not have relevance in the new repo.
 		// Use Original as a persistent snapshot ID
 		if sn.Original == nil {
 			sn.Original = sn.ID()
 		}
-		newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+		newID, err := restic.SaveSnapshot(ctx, dstRepo, sn)
 		if err != nil {
 			return err
 		}
@@ -175,66 +187,60 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 }
 
 func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
-	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
+	visitedTrees restic.IDSet, rootTreeID restic.ID, quiet bool) error {
 
-	wg, ctx := errgroup.WithContext(ctx)
+	wg, wgCtx := errgroup.WithContext(ctx)
 
-	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
+	treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
 		visited := visitedTrees.Has(treeID)
 		visitedTrees.Insert(treeID)
 		return visited
 	}, nil)
 
-	wg.Go(func() error {
-		// reused buffer
-		var buf []byte
+	copyBlobs := restic.NewBlobSet()
+	packList := restic.NewIDSet()
 
+	enqueue := func(h restic.BlobHandle) {
+		pb := srcRepo.Index().Lookup(h)
+		copyBlobs.Insert(h)
+		for _, p := range pb {
+			packList.Insert(p.PackID)
+		}
+	}
+
+	wg.Go(func() error {
 		for tree := range treeStream {
 			if tree.Error != nil {
 				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
 			}
 
 			// Do we already have this tree blob?
-			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
+			treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
+			if !dstRepo.Index().Has(treeHandle) {
 				// copy raw tree bytes to avoid problems if the serialization changes
-				var err error
-				buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
-				if err != nil {
-					return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
-				}
-
-				_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
-				if err != nil {
-					return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
-				}
+				enqueue(treeHandle)
 			}
 
-			// TODO: parallelize blob down/upload
-
 			for _, entry := range tree.Nodes {
 				// Recursion into directories is handled by StreamTrees
 				// Copy the blobs for this file.
 				for _, blobID := range entry.Content {
-					// Do we already have this data blob?
-					if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
-						continue
-					}
-					debug.Log("Copying blob %s\n", blobID.Str())
-					var err error
-					buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
-					if err != nil {
-						return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
-					}
-
-					_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
-					if err != nil {
-						return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
+					h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
+					if !dstRepo.Index().Has(h) {
+						enqueue(h)
 					}
 				}
 			}
-
 		}
 		return nil
 	})
-	return wg.Wait()
+	err := wg.Wait()
+	if err != nil {
+		return err
+	}
+
+	bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
+	_, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar)
+	bar.Done()
+	return err
 }

cmd/restic/cmd_debug.go

@@ -1,3 +1,4 @@
+//go:build debug
 // +build debug
 
 package main
@@ -14,6 +15,7 @@ import (
 	"sort"
 	"time"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 
@@ -51,12 +53,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 var tryRepair bool
 var repairByte bool
 var extractPack bool
+var reuploadBlobs bool
 
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
 	cmdDebug.AddCommand(cmdDebugExamine)
 	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
+	cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
 	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
 	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
@@ -72,7 +76,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 }
 
 func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
-	return restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
+	return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
 		if err != nil {
 			return err
 		}
@@ -103,7 +107,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 	return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 
-		blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
+		blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
 			return nil
@@ -308,6 +312,10 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
 }
 
 func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
+	dec, err := zstd.NewReader(nil)
+	if err != nil {
+		panic(err)
+	}
 	be := repo.Backend()
 	h := restic.Handle{
 		Name: pack.String(),
@@ -332,48 +340,65 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list
 
 		nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
 		plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+		outputPrefix := ""
+		filePrefix := ""
 		if err != nil {
 			Warnf("error decrypting blob: %v\n", err)
-			var plain []byte
 			if tryRepair || repairByte {
-				plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
+				plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
 			}
-			var prefix string
-			if plain != nil {
-				id := restic.Hash(plain)
-				if !id.Equal(blob.ID) {
-					Printf("         repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
-					prefix = "repaired-wrong-hash-"
-				} else {
-					Printf("         successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
-					prefix = "repaired-"
-				}
+			if plaintext != nil {
+				outputPrefix = "repaired "
+				filePrefix = "repaired-"
 			} else {
-				plain = decryptUnsigned(ctx, key, buf)
-				prefix = "damaged-"
+				plaintext = decryptUnsigned(ctx, key, buf)
+				err = storePlainBlob(blob.ID, "damaged-", plaintext)
+				if err != nil {
+					return err
+				}
+				continue
 			}
-			err = storePlainBlob(blob.ID, prefix, plain)
+		}
+
+		if blob.IsCompressed() {
+			decompressed, err := dec.DecodeAll(plaintext, nil)
 			if err != nil {
-				return err
+				Printf("         failed to decompress blob %v\n", blob.ID)
+			}
+			if decompressed != nil {
+				plaintext = decompressed
 			}
-			continue
 		}
 
 		id := restic.Hash(plaintext)
 		var prefix string
 		if !id.Equal(blob.ID) {
-			Printf("         successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
+			Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
 			prefix = "wrong-hash-"
 		} else {
-			Printf("         successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
+			Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
 			prefix = "correct-"
 		}
 		if extractPack {
-			err = storePlainBlob(id, prefix, plaintext)
+			err = storePlainBlob(id, filePrefix+prefix, plaintext)
 			if err != nil {
 				return err
 			}
 		}
+		if reuploadBlobs {
+			_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
+			if err != nil {
+				return err
+			}
+			Printf("         uploaded %v %v\n", blob.Type, id)
+		}
+	}
+
+	if reuploadBlobs {
+		err := repo.Flush(ctx)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -402,12 +427,23 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 }
 
 func runDebugExamine(gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
 	ids := make([]restic.ID, 0)
 	for _, name := range args {
 		id, err := restic.ParseID(name)
 		if err != nil {
-			Warnf("error: %v\n", err)
-			continue
+			name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name)
+			if err == nil {
+				id, err = restic.ParseID(name)
+			}
+			if err != nil {
+				Warnf("error: %v\n", err)
+				continue
+			}
 		}
 		ids = append(ids, id)
 	}
@@ -416,11 +452,6 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no pack files to examine")
 	}
 
-	repo, err := OpenRepository(gopts)
-	if err != nil {
-		return err
-	}
-
 	if !gopts.NoLock {
 		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
@@ -475,27 +506,15 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 
 	blobsLoaded := false
 	// examine all data the indexes have for the pack file
-	for _, idx := range repo.Index().(*repository.MasterIndex).All() {
-		idxIDs, err := idx.IDs()
-		if err != nil {
-			idxIDs = restic.IDs{}
-		}
-
-		blobs := idx.ListPack(id)
+	for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) {
+		blobs := b.Blobs
 		if len(blobs) == 0 {
 			continue
 		}
 
-		Printf("    index %v:\n", idxIDs)
+		checkPackSize(blobs, fi.Size)
 
-		// convert list of blobs to []restic.Blob
-		var list []restic.Blob
-		for _, b := range blobs {
-			list = append(list, b.Blob)
-		}
-		checkPackSize(list, fi.Size)
-
-		err = loadBlobs(ctx, repo, id, list)
+		err = loadBlobs(ctx, repo, id, blobs)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		} else {
@@ -506,7 +525,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 	Printf("  ========================================\n")
 	Printf("  inspect the pack itself\n")
 
-	blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
+	blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size)
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}
@@ -531,14 +550,10 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) {
 		if offset != uint64(pb.Offset) {
 			Printf("      hole in file, want offset %v, got %v\n", offset, pb.Offset)
 		}
-		offset += uint64(pb.Length)
+		offset = uint64(pb.Offset + pb.Length)
 		size += uint64(pb.Length)
 	}
-
-	// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
-	size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
-	// length in uint32 little endian
-	size += 4
+	size += uint64(pack.CalculateHeaderSize(blobs))
 
 	if uint64(fileSize) != size {
 		Printf("      file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)

cmd/restic/cmd_diff.go

@@ -2,13 +2,14 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"path"
 	"reflect"
 	"sort"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -52,8 +53,8 @@ func init() {
 	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
-func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
-	id, err := restic.FindSnapshot(ctx, repo, desc)
+func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
+	id, err := restic.FindSnapshot(ctx, be, desc)
 	if err != nil {
 		return nil, errors.Fatal(err.Error())
 	}
@@ -62,15 +63,29 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string)
 
 // Comparer collects all things needed to compare two snapshots.
 type Comparer struct {
-	repo restic.Repository
-	opts DiffOptions
+	repo        restic.Repository
+	opts        DiffOptions
+	printChange func(change *Change)
+}
+
+type Change struct {
+	MessageType string `json:"message_type"` // "change"
+	Path        string `json:"path"`
+	Modifier    string `json:"modifier"`
+}
+
+func NewChange(path string, mode string) *Change {
+	return &Change{MessageType: "change", Path: path, Modifier: mode}
 }
 
 // DiffStat collects stats for all types of items.
 type DiffStat struct {
-	Files, Dirs, Others  int
-	DataBlobs, TreeBlobs int
-	Bytes                uint64
+	Files     int    `json:"files"`
+	Dirs      int    `json:"dirs"`
+	Others    int    `json:"others"`
+	DataBlobs int    `json:"data_blobs"`
+	TreeBlobs int    `json:"tree_blobs"`
+	Bytes     uint64 `json:"bytes"`
 }
 
 // Add adds stats information for node to s.
@@ -113,21 +128,14 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 	}
 }
 
-// DiffStats collects the differences between two snapshots.
-type DiffStats struct {
-	ChangedFiles                         int
-	Added                                DiffStat
-	Removed                              DiffStat
-	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
-}
-
-// NewDiffStats creates new stats for a diff run.
-func NewDiffStats() *DiffStats {
-	return &DiffStats{
-		BlobsBefore: restic.NewBlobSet(),
-		BlobsAfter:  restic.NewBlobSet(),
-		BlobsCommon: restic.NewBlobSet(),
-	}
+type DiffStatsContainer struct {
+	MessageType                          string         `json:"message_type"` // "statistics"
+	SourceSnapshot                       string         `json:"source_snapshot"`
+	TargetSnapshot                       string         `json:"target_snapshot"`
+	ChangedFiles                         int            `json:"changed_files"`
+	Added                                DiffStat       `json:"added"`
+	Removed                              DiffStat       `json:"removed"`
+	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
 }
 
 // updateBlobs updates the blob counters in the stats struct.
@@ -152,7 +160,7 @@ func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat)
 
 func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
 	debug.Log("print %v tree %v", mode, id)
-	tree, err := c.repo.LoadTree(ctx, id)
+	tree, err := restic.LoadTree(ctx, c.repo, id)
 	if err != nil {
 		return err
 	}
@@ -162,7 +170,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		if node.Type == "dir" {
 			name += "/"
 		}
-		Printf("%-5s%v\n", mode, name)
+		c.printChange(NewChange(name, mode))
 		stats.Add(node)
 		addBlobs(blobs, node)
 
@@ -179,7 +187,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 
 func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
 	debug.Log("print tree %v", id)
-	tree, err := c.repo.LoadTree(ctx, id)
+	tree, err := restic.LoadTree(ctx, c.repo, id)
 	if err != nil {
 		return err
 	}
@@ -221,14 +229,14 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
 	return tree1Nodes, tree2Nodes, uniqueNames
 }
 
-func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
+func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
-	tree1, err := c.repo.LoadTree(ctx, id1)
+	tree1, err := restic.LoadTree(ctx, c.repo, id1)
 	if err != nil {
 		return err
 	}
 
-	tree2, err := c.repo.LoadTree(ctx, id2)
+	tree2, err := restic.LoadTree(ctx, c.repo, id2)
 	if err != nil {
 		return err
 	}
@@ -265,7 +273,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			}
 
 			if mod != "" {
-				Printf("%-5s%v\n", mod, name)
+				c.printChange(NewChange(name, mod))
 			}
 
 			if node1.Type == "dir" && node2.Type == "dir" {
@@ -284,7 +292,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node1.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "-", prefix)
+			c.printChange(NewChange(prefix, "-"))
 			stats.Removed.Add(node1)
 
 			if node1.Type == "dir" {
@@ -298,7 +306,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node2.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "+", prefix)
+			c.printChange(NewChange(prefix, "+"))
 			stats.Added.Add(node2)
 
 			if node2.Type == "dir" {
@@ -326,10 +334,6 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if err = repo.LoadIndex(ctx); err != nil {
-		return err
-	}
-
 	if !gopts.NoLock {
 		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
@@ -338,17 +342,28 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	sn1, err := loadSnapshot(ctx, repo, args[0])
+	// cache snapshots listing
+	be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+	sn1, err := loadSnapshot(ctx, be, repo, args[0])
 	if err != nil {
 		return err
 	}
 
-	sn2, err := loadSnapshot(ctx, repo, args[1])
+	sn2, err := loadSnapshot(ctx, be, repo, args[1])
 	if err != nil {
 		return err
 	}
 
-	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	if !gopts.JSON {
+		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	}
+
+	if err = repo.LoadIndex(ctx); err != nil {
+		return err
+	}
 
 	if sn1.Tree == nil {
 		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
@@ -361,9 +376,33 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,
+		printChange: func(change *Change) {
+			Printf("%-5s%v\n", change.Modifier, change.Path)
+		},
 	}
 
-	stats := NewDiffStats()
+	if gopts.JSON {
+		enc := json.NewEncoder(gopts.stdout)
+		c.printChange = func(change *Change) {
+			err := enc.Encode(change)
+			if err != nil {
+				Warnf("JSON encode failed: %v\n", err)
+			}
+		}
+	}
+
+	if gopts.Quiet {
+		c.printChange = func(change *Change) {}
+	}
+
+	stats := &DiffStatsContainer{
+		MessageType:    "statistics",
+		SourceSnapshot: args[0],
+		TargetSnapshot: args[1],
+		BlobsBefore:    restic.NewBlobSet(),
+		BlobsAfter:     restic.NewBlobSet(),
+		BlobsCommon:    restic.NewBlobSet(),
+	}
 	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
 	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
 
@@ -376,14 +415,21 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
 	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
 
-	Printf("\n")
-	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
-	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
-	Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
-	Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
-	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
-	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
-	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+	if gopts.JSON {
+		err := json.NewEncoder(gopts.stdout).Encode(stats)
+		if err != nil {
+			Warnf("JSON encode failed: %v\n", err)
+		}
+	} else {
+		Printf("\n")
+		Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
+		Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
+		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
+		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
+		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
+		Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
+		Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+	}
 
 	return nil
 }

cmd/restic/cmd_dump.go

@@ -67,50 +67,40 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
-	if tree == nil {
-		return fmt.Errorf("called with a nil tree")
-	}
-	if repo == nil {
-		return fmt.Errorf("called with a nil repository")
-	}
-	l := len(pathComponents)
-	if l == 0 {
-		return fmt.Errorf("empty path components")
-	}
-
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
 		if err := checkStdoutArchive(); err != nil {
 			return err
 		}
-		return writeDump(ctx, repo, tree, "/", os.Stdout)
+		return d.DumpTree(ctx, tree, "/")
 	}
 
 	item := filepath.Join(prefix, pathComponents[0])
+	l := len(pathComponents)
 	for _, node := range tree.Nodes {
 		// If dumping something in the highest level it will just take the
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {
 			switch {
 			case l == 1 && dump.IsFile(node):
-				return dump.GetNodeData(ctx, os.Stdout, repo, node)
+				return d.WriteNode(ctx, node)
 			case l > 1 && dump.IsDir(node):
-				subtree, err := repo.LoadTree(ctx, *node.Subtree)
+				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
 			case dump.IsDir(node):
 				if err := checkStdoutArchive(); err != nil {
 					return err
 				}
-				subtree, err := repo.LoadTree(ctx, *node.Subtree)
+				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
 				if err != nil {
 					return err
 				}
-				return writeDump(ctx, repo, subtree, item, os.Stdout)
+				return d.DumpTree(ctx, subtree, item)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
 			case !dump.IsFile(node):
@@ -128,12 +118,8 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 
-	var wd dump.WriteDump
 	switch opts.Archive {
-	case "tar":
-		wd = dump.WriteTar
-	case "zip":
-		wd = dump.WriteZip
+	case "tar", "zip":
 	default:
 		return fmt.Errorf("unknown archive format %q", opts.Archive)
 	}
@@ -158,20 +144,15 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex(ctx)
-	if err != nil {
-		return err
-	}
-
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
+		id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
-		id, err = restic.FindSnapshot(ctx, repo, snapshotIDString)
+		id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString)
 		if err != nil {
 			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
 		}
@@ -182,12 +163,18 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	tree, err := repo.LoadTree(ctx, *sn.Tree)
+	err = repo.LoadIndex(ctx)
+	if err != nil {
+		return err
+	}
+
+	tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
 	if err != nil {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
+	d := dump.New(opts.Archive, repo, os.Stdout)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
 	if err != nil {
 		Exitf(2, "cannot dump file: %v", err)
 	}

cmd/restic/cmd_find.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
@@ -267,7 +268,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}
@@ -351,7 +352,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}
@@ -584,6 +585,11 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	if err = repo.LoadIndex(gopts.ctx); err != nil {
 		return err
 	}
@@ -618,7 +624,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
 		if f.blobIDs != nil || f.treeIDs != nil {
 			if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
 				return err

cmd/restic/cmd_forget.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"io"
 
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -13,10 +14,16 @@ var cmdForget = &cobra.Command{
 	Use:   "forget [flags] [snapshot ID] [...]",
 	Short: "Remove snapshots from the repository",
 	Long: `
-The "forget" command removes snapshots according to a policy. Please note that
-this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove this (now unreferenced)
-data after 'forget' was run successfully, see the 'prune' command.
+The "forget" command removes snapshots according to a policy. All snapshots are
+first divided into groups according to "--group-by", and after that the policy
+specified by the "--keep-*" options is applied to each group individually.
+
+Please note that this command really only deletes the snapshot object in the
+repository, which is a reference to data stored there. In order to remove the
+unreferenced data after "forget" was run successfully, see the "prune" command.
+
+Please also read the documentation for "forget" to learn about some important
+security considerations.
 
 EXIT STATUS
 ===========
@@ -89,7 +96,7 @@ func init() {
 	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
 
-	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "string for grouping snapshots by host,paths,tags")
+	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
@@ -108,10 +115,16 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(gopts.ctx, repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
+	if gopts.NoLock && !opts.DryRun {
+		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
+	}
+
+	if !opts.DryRun || !gopts.NoLock {
+		lock, err := lockRepoExclusive(gopts.ctx, repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
 	}
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
@@ -120,7 +133,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		snapshots = append(snapshots, sn)
 	}
 

cmd/restic/cmd_init.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"strconv"
+
 	"github.com/restic/chunker"
 	"github.com/restic/restic/internal/backend/location"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 )
@@ -30,6 +33,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type InitOptions struct {
 	secondaryRepoOptions
 	CopyChunkerParameters bool
+	RepositoryVersion     string
 }
 
 var initOptions InitOptions
@@ -40,9 +44,26 @@ func init() {
 	f := cmdInit.Flags()
 	initSecondaryRepoOptions(f, &initOptions.secondaryRepoOptions, "secondary", "to copy chunker parameters from")
 	f.BoolVar(&initOptions.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)")
+	f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
 }
 
 func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
+	var version uint
+	if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
+		version = restic.MaxRepoVersion
+	} else if opts.RepositoryVersion == "stable" {
+		version = restic.StableRepoVersion
+	} else {
+		v, err := strconv.ParseUint(opts.RepositoryVersion, 10, 32)
+		if err != nil {
+			return errors.Fatal("invalid repository version")
+		}
+		version = uint(v)
+	}
+	if version < restic.MinRepoVersion || version > restic.MaxRepoVersion {
+		return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
+	}
+
 	chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
 	if err != nil {
 		return err
@@ -65,9 +86,15 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 
-	s := repository.New(be)
+	s, err := repository.New(be, repository.Options{
+		Compression: gopts.Compression,
+		PackSize:    gopts.PackSize * 1024 * 1024,
+	})
+	if err != nil {
+		return err
+	}
 
-	err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)
+	err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial)
 	if err != nil {
 		return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
@@ -83,7 +110,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 
 func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
 	if opts.CopyChunkerParameters {
-		otherGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
+		otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
 		if err != nil {
 			return nil, err
 		}
@@ -97,7 +124,7 @@ func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker
 		return &pol, nil
 	}
 
-	if opts.Repo != "" {
+	if opts.Repo != "" || opts.RepositoryFile != "" || opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" {
 		return nil, errors.Fatal("Secondary repository must only be specified when copying the chunker parameters")
 	}
 	return nil, nil

cmd/restic/cmd_key.go

@@ -131,6 +131,11 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
+	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
 	Verbosef("saved new key as %s\n", id)
 
 	return nil
@@ -161,8 +166,14 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
+	oldID := repo.KeyName()
 
-	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
+	h := restic.Handle{Type: restic.KeyFile, Name: oldID}
 	err = repo.Backend().Remove(gopts.ctx, h)
 	if err != nil {
 		return err
@@ -173,6 +184,19 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	return nil
 }
 
+func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
+	// Verify new key to make sure it really works. A broken key can render the
+	// whole repository inaccessible
+	err := repo.SearchKey(ctx, pw, 0, key.Name())
+	if err != nil {
+		// the key is invalid, try to remove it
+		h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
+		_ = repo.Backend().Remove(ctx, h)
+		return errors.Fatalf("failed to access repository with new key: %v", err)
+	}
+	return nil
+}
+
 func runKey(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
 		return errors.Fatal("wrong number of arguments")

cmd/restic/cmd_list.go

@@ -39,7 +39,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if !opts.NoLock {
+	if !opts.NoLock && args[0] != "locks" {
 		lock, err := lockRepo(opts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {

cmd/restic/cmd_ls.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -61,9 +62,9 @@ func init() {
 
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
-	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 }
 
@@ -77,31 +78,33 @@ type lsSnapshot struct {
 // Print node in our custom JSON format, followed by a newline.
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	n := &struct {
-		Name       string      `json:"name"`
-		Type       string      `json:"type"`
-		Path       string      `json:"path"`
-		UID        uint32      `json:"uid"`
-		GID        uint32      `json:"gid"`
-		Size       *uint64     `json:"size,omitempty"`
-		Mode       os.FileMode `json:"mode,omitempty"`
-		ModTime    time.Time   `json:"mtime,omitempty"`
-		AccessTime time.Time   `json:"atime,omitempty"`
-		ChangeTime time.Time   `json:"ctime,omitempty"`
-		StructType string      `json:"struct_type"` // "node"
+		Name        string      `json:"name"`
+		Type        string      `json:"type"`
+		Path        string      `json:"path"`
+		UID         uint32      `json:"uid"`
+		GID         uint32      `json:"gid"`
+		Size        *uint64     `json:"size,omitempty"`
+		Mode        os.FileMode `json:"mode,omitempty"`
+		Permissions string      `json:"permissions,omitempty"`
+		ModTime     time.Time   `json:"mtime,omitempty"`
+		AccessTime  time.Time   `json:"atime,omitempty"`
+		ChangeTime  time.Time   `json:"ctime,omitempty"`
+		StructType  string      `json:"struct_type"` // "node"
 
 		size uint64 // Target for Size pointer.
 	}{
-		Name:       node.Name,
-		Type:       node.Type,
-		Path:       path,
-		UID:        node.UID,
-		GID:        node.GID,
-		size:       node.Size,
-		Mode:       node.Mode,
-		ModTime:    node.ModTime,
-		AccessTime: node.AccessTime,
-		ChangeTime: node.ChangeTime,
-		StructType: "node",
+		Name:        node.Name,
+		Type:        node.Type,
+		Path:        path,
+		UID:         node.UID,
+		GID:         node.GID,
+		size:        node.Size,
+		Mode:        node.Mode,
+		Permissions: node.Mode.String(),
+		ModTime:     node.ModTime,
+		AccessTime:  node.AccessTime,
+		ChangeTime:  node.ChangeTime,
+		StructType:  "node",
 	}
 	// Always print size for regular files, even when empty,
 	// but never for other types.
@@ -114,7 +117,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatal("no snapshot ID specified")
+		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
 
 	// extract any specific directories to walk
@@ -167,6 +170,11 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
+	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	if err = repo.LoadIndex(gopts.ctx); err != nil {
 		return err
 	}
@@ -209,7 +217,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
 		printSnapshot(sn)
 
 		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {

cmd/restic/cmd_ls_test.go

@@ -18,6 +18,7 @@ func TestLsNodeJSON(t *testing.T) {
 		expect string
 	}{
 		// Mode is omitted when zero.
+		// Permissions, by convention is "-" per mode bit
 		{
 			path: "/bar/baz",
 			Node: restic.Node{
@@ -31,7 +32,7 @@ func TestLsNodeJSON(t *testing.T) {
 				Group: "nobodies",
 				Links: 1,
 			},
-			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 
 		// Even empty files get an explicit size.
@@ -48,7 +49,7 @@ func TestLsNodeJSON(t *testing.T) {
 				Group: "not printed",
 				Links: 0xF00,
 			},
-			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 
 		// Non-regular files do not get a size.
@@ -61,7 +62,7 @@ func TestLsNodeJSON(t *testing.T) {
 				Mode:       os.ModeSymlink | 0777,
 				LinkTarget: "not printed",
 			},
-			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 
 		{
@@ -74,7 +75,7 @@ func TestLsNodeJSON(t *testing.T) {
 				AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
 				ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
 			},
-			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
+			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
 		},
 	} {
 		buf := new(bytes.Buffer)

cmd/restic/cmd_migrate.go

@@ -8,11 +8,12 @@ import (
 )
 
 var cmdMigrate = &cobra.Command{
-	Use:   "migrate [flags] [name]",
+	Use:   "migrate [flags] [migration name] [...]",
 	Short: "Apply migrations",
 	Long: `
-The "migrate" command applies migrations to a repository. When no migration
-name is explicitly given, a list of migrations that can be applied is printed.
+The "migrate" command checks which migrations can be applied for a repository
+and prints a list with available migration names. If one or more migration
+names are specified, these migrations are applied.
 
 EXIT STATUS
 ===========
@@ -41,6 +42,8 @@ func init() {
 func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
 	ctx := gopts.ctx
 	Printf("available migrations:\n")
+	found := false
+
 	for _, m := range migrations.All {
 		ok, err := m.Check(ctx, repo)
 		if err != nil {
@@ -48,10 +51,15 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 		}
 
 		if ok {
-			Printf("  %v: %v\n", m.Name(), m.Desc())
+			Printf("  %v\t%v\n", m.Name(), m.Desc())
+			found = true
 		}
 	}
 
+	if !found {
+		Printf("no migrations found")
+	}
+
 	return nil
 }
 
@@ -76,6 +84,19 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 					Warnf("check for migration %v failed, continuing anyway\n", m.Name())
 				}
 
+				if m.RepoCheck() {
+					Printf("checking repository integrity...\n")
+
+					checkOptions := CheckOptions{}
+					checkGopts := gopts
+					// the repository is already locked
+					checkGopts.NoLock = true
+					err = runCheck(checkOptions, checkGopts, []string{})
+					if err != nil {
+						return err
+					}
+				}
+
 				Printf("applying migration %v...\n", m.Name())
 				if err = m.Apply(ctx, repo); err != nil {
 					Warnf("migration %v failed: %v\n", m.Name(), err)

cmd/restic/cmd_mount.go

@@ -1,3 +1,4 @@
+//go:build darwin || freebsd || linux
 // +build darwin freebsd linux
 
 package main
@@ -30,10 +31,13 @@ read-only mount.
 Snapshot Directories
 ====================
 
-If you need a different template for all directories that contain snapshots,
-you can pass a template via --snapshot-template. Example without colons:
+If you need a different template for directories that contain snapshots,
+you can pass a time template via --time-template and path templates via
+--path-template.
 
-    --snapshot-template "2006-01-02_15-04-05"
+Example time template without colons:
+
+    --time-template "2006-01-02_15-04-05"
 
 You need to specify a sample format for exactly the following timestamp:
 
@@ -42,6 +46,20 @@ You need to specify a sample format for exactly the following timestamp:
 For details please see the documentation for time.Format() at:
   https://godoc.org/time#Time.Format
 
+For path templates, you can use the following patterns which will be replaced:
+    %i by short snapshot ID
+    %I by long snapshot ID
+    %u by username
+    %h by hostname
+    %t by tags
+    %T by timestamp as specified by --time-template
+
+The default path templates are:
+    "ids/%i"
+    "snapshots/%T"
+    "hosts/%h/%T"
+    "tags/%t/%T"
+
 EXIT STATUS
 ===========
 
@@ -61,7 +79,8 @@ type MountOptions struct {
 	Hosts                []string
 	Tags                 restic.TagLists
 	Paths                []string
-	SnapshotTemplate     string
+	TimeTemplate         string
+	PathTemplates        []string
 }
 
 var mountOptions MountOptions
@@ -78,16 +97,21 @@ func init() {
 	mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
 	mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 
-	mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
+	mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
+	mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
+	mountFlags.StringVar(&mountOptions.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times")
+	_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
 }
 
 func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
-	if opts.SnapshotTemplate == "" {
-		return errors.Fatal("snapshot template string cannot be empty")
+	if opts.TimeTemplate == "" {
+		return errors.Fatal("time template string cannot be empty")
 	}
-	if strings.ContainsAny(opts.SnapshotTemplate, `\/`) {
-		return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character")
+
+	if strings.HasPrefix(opts.TimeTemplate, "/") || strings.HasSuffix(opts.TimeTemplate, "/") {
+		return errors.Fatal("time template string cannot start or end with '/'")
 	}
+
 	if len(args) == 0 {
 		return errors.Fatal("wrong number of parameters")
 	}
@@ -115,7 +139,7 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 
 	mountpoint := args[0]
 
-	if _, err := resticfs.Stat(mountpoint); os.IsNotExist(errors.Cause(err)) {
+	if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
 		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
 		return err
 	}
@@ -153,16 +177,18 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	cfg := fuse.Config{
-		OwnerIsRoot:      opts.OwnerRoot,
-		Hosts:            opts.Hosts,
-		Tags:             opts.Tags,
-		Paths:            opts.Paths,
-		SnapshotTemplate: opts.SnapshotTemplate,
+		OwnerIsRoot:   opts.OwnerRoot,
+		Hosts:         opts.Hosts,
+		Tags:          opts.Tags,
+		Paths:         opts.Paths,
+		TimeTemplate:  opts.TimeTemplate,
+		PathTemplates: opts.PathTemplates,
 	}
 	root := fuse.NewRoot(repo, cfg)
 
 	Printf("Now serving the repository at %s\n", mountpoint)
-	Printf("When finished, quit with Ctrl-c or umount the mountpoint.\n")
+	Printf("Use another terminal or tool to browse the contents of this folder.\n")
+	Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
 
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, root)

cmd/restic/cmd_prune.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"math"
 	"sort"
 	"strconv"
@@ -8,6 +9,7 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 
@@ -38,7 +40,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // PruneOptions collects all options for the cleanup command.
 type PruneOptions struct {
-	DryRun bool
+	DryRun                bool
+	UnsafeNoSpaceRecovery string
+
+	unsafeRecovery bool
 
 	MaxUnused      string
 	maxUnusedBytes func(used uint64) (unused uint64) // calculates the number of unused bytes after repacking, according to MaxUnused
@@ -47,6 +52,8 @@ type PruneOptions struct {
 	MaxRepackBytes uint64
 
 	RepackCachableOnly bool
+	RepackSmall        bool
+	RepackUncompressed bool
 }
 
 var pruneOptions PruneOptions
@@ -55,6 +62,7 @@ func init() {
 	cmdRoot.AddCommand(cmdPrune)
 	f := cmdPrune.Flags()
 	f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
+	f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
 	addPruneOptions(cmdPrune)
 }
 
@@ -63,9 +71,12 @@ func addPruneOptions(c *cobra.Command) {
 	f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
 	f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
 	f.BoolVar(&pruneOptions.RepackCachableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
+	f.BoolVar(&pruneOptions.RepackSmall, "repack-small", false, "repack pack files below 80% of target pack size")
+	f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data")
 }
 
 func verifyPruneOptions(opts *PruneOptions) error {
+	opts.MaxRepackBytes = math.MaxUint64
 	if len(opts.MaxRepackSize) > 0 {
 		size, err := parseSizeStr(opts.MaxRepackSize)
 		if err != nil {
@@ -73,6 +84,10 @@ func verifyPruneOptions(opts *PruneOptions) error {
 		}
 		opts.MaxRepackBytes = uint64(size)
 	}
+	if opts.UnsafeNoSpaceRecovery != "" {
+		// prevent repacking data to make sure users cannot get stuck.
+		opts.MaxRepackBytes = 0
+	}
 
 	maxUnused := strings.TrimSpace(opts.MaxUnused)
 	if maxUnused == "" {
@@ -119,29 +134,37 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 
-func shortenStatus(maxLength int, s string) string {
-	if len(s) <= maxLength {
-		return s
-	}
-
-	if maxLength < 3 {
-		return s[:maxLength]
-	}
-
-	return s[:maxLength-3] + "..."
-}
-
 func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
 		return err
 	}
 
+	if opts.RepackUncompressed && gopts.Compression == repository.CompressionOff {
+		return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
 	}
 
+	if repo.Backend().Connections() < 2 {
+		return errors.Fatal("prune requires a backend connection limit of at least two")
+	}
+
+	if repo.Config().Version < 2 && opts.RepackUncompressed {
+		return errors.Fatal("compression requires at least repository format version 2")
+	}
+
+	if opts.UnsafeNoSpaceRecovery != "" {
+		repoID := repo.Config().ID
+		if opts.UnsafeNoSpaceRecovery != repoID {
+			return errors.Fatalf("must pass id '%s' to --unsafe-recover-no-free-space", repoID)
+		}
+		opts.unsafeRecovery = true
+	}
+
 	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
@@ -160,26 +183,69 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
 	}
 
 	Verbosef("loading indexes...\n")
+	// loading the index before the snapshots is ok, as we use an exclusive lock here
 	err := repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 
-	usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
+	plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots)
 	if err != nil {
 		return err
 	}
 
-	return prune(opts, gopts, repo, usedBlobs)
+	err = printPruneStats(gopts, stats)
+	if err != nil {
+		return err
+	}
+
+	return doPrune(opts, gopts, repo, plan)
+}
+
+type pruneStats struct {
+	blobs struct {
+		used      uint
+		duplicate uint
+		unused    uint
+		remove    uint
+		repack    uint
+		repackrm  uint
+	}
+	size struct {
+		used      uint64
+		duplicate uint64
+		unused    uint64
+		remove    uint64
+		repack    uint64
+		repackrm  uint64
+		unref     uint64
+	}
+	packs struct {
+		used       uint
+		unused     uint
+		partlyUsed uint
+		unref      uint
+		keep       uint
+		repack     uint
+		remove     uint
+	}
+}
+
+type prunePlan struct {
+	removePacksFirst restic.IDSet   // packs to remove first (unreferenced packs)
+	repackPacks      restic.IDSet   // packs to repack
+	keepBlobs        restic.BlobSet // blobs to keep during repacking
+	removePacks      restic.IDSet   // packs to remove
+	ignorePacks      restic.IDSet   // packs to ignore when rebuilding the index
 }
 
 type packInfo struct {
-	usedBlobs      uint
-	unusedBlobs    uint
-	duplicateBlobs uint
-	usedSize       uint64
-	unusedSize     uint64
-	tpe            restic.BlobType
+	usedBlobs    uint
+	unusedBlobs  uint
+	usedSize     uint64
+	unusedSize   uint64
+	tpe          restic.BlobType
+	uncompressed bool
 }
 
 type packInfoWithID struct {
@@ -187,44 +253,53 @@ type packInfoWithID struct {
 	packInfo
 }
 
-// prune selects which files to rewrite and then does that. The map usedBlobs is
-// modified in the process.
-func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedBlobs restic.BlobSet) error {
+// planPrune selects which files to rewrite and which to delete and which blobs to keep.
+// Also some summary statistics are returned.
+func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) {
 	ctx := gopts.ctx
+	var stats pruneStats
 
-	var stats struct {
-		blobs struct {
-			used      uint
-			duplicate uint
-			unused    uint
-			remove    uint
-			repack    uint
-			repackrm  uint
-		}
-		size struct {
-			used      uint64
-			duplicate uint64
-			unused    uint64
-			remove    uint64
-			repack    uint64
-			repackrm  uint64
-			unref     uint64
-		}
-		packs struct {
-			used       uint
-			unused     uint
-			partlyUsed uint
-			keep       uint
-		}
+	usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
+	if err != nil {
+		return prunePlan{}, stats, err
 	}
 
 	Verbosef("searching used packs...\n")
+	keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats)
+	if err != nil {
+		return prunePlan{}, stats, err
+	}
 
+	Verbosef("collecting packs for deletion and repacking\n")
+	plan, err := decidePackAction(ctx, opts, gopts, repo, indexPack, &stats)
+	if err != nil {
+		return prunePlan{}, stats, err
+	}
+
+	if len(plan.repackPacks) != 0 {
+		// when repacking, we do not want to keep blobs which are
+		// already contained in kept packs, so delete them from keepBlobs
+		for blob := range repo.Index().Each(ctx) {
+			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
+				continue
+			}
+			keepBlobs.Delete(blob.BlobHandle)
+		}
+	} else {
+		// keepBlobs is only needed if packs are repacked
+		keepBlobs = nil
+	}
+	plan.keepBlobs = keepBlobs
+
+	return plan, stats, nil
+}
+
+func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.BlobSet, stats *pruneStats) (restic.BlobSet, map[restic.ID]packInfo, error) {
 	keepBlobs := restic.NewBlobSet()
-	duplicateBlobs := restic.NewBlobSet()
+	duplicateBlobs := make(map[restic.BlobHandle]uint8)
 
 	// iterate over all blobs in index to find out which blobs are duplicates
-	for blob := range repo.Index().Each(ctx) {
+	for blob := range idx.Each(ctx) {
 		bh := blob.BlobHandle
 		size := uint64(blob.Length)
 		switch {
@@ -234,7 +309,16 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			stats.size.used += size
 			stats.blobs.used++
 		case keepBlobs.Has(bh): // duplicate blob
-			duplicateBlobs.Insert(bh)
+			count, ok := duplicateBlobs[bh]
+			if !ok {
+				count = 2 // this one is already the second blob!
+			} else if count < math.MaxUint8 {
+				// don't overflow, but saturate count at 255
+				// this can lead to a non-optimal pack selection, but won't cause
+				// problems otherwise
+				count++
+			}
+			duplicateBlobs[bh] = count
 			stats.size.duplicate += size
 			stats.blobs.duplicate++
 		default:
@@ -250,19 +334,19 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			"Will not start prune to prevent (additional) data loss!\n"+
 			"Please report this error (along with the output of the 'prune' run) at\n"+
 			"https://github.com/restic/restic/issues/new/choose\n", usedBlobs)
-		return errorIndexIncomplete
+		return nil, nil, errorIndexIncomplete
 	}
 
 	indexPack := make(map[restic.ID]packInfo)
 
 	// save computed pack header size
-	for pid, hdrSize := range repo.Index().PackSize(ctx, true) {
+	for pid, hdrSize := range pack.Size(ctx, idx, true) {
 		// initialize tpe with NumBlobTypes to indicate it's not set
 		indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
 	}
 
 	// iterate over all blobs in index to generate packInfo
-	for blob := range repo.Index().Each(ctx) {
+	for blob := range idx.Each(ctx) {
 		ip := indexPack[blob.PackID]
 
 		// Set blob type if not yet set
@@ -277,10 +361,9 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 		bh := blob.BlobHandle
 		size := uint64(blob.Length)
+		_, isDuplicate := duplicateBlobs[bh]
 		switch {
-		case duplicateBlobs.Has(bh): // duplicate blob
-			ip.usedSize += size
-			ip.duplicateBlobs++
+		case isDuplicate: // duplicate blobs will be handled later
 		case keepBlobs.Has(bh): // used blob, not duplicate
 			ip.usedSize += size
 			ip.usedBlobs++
@@ -288,23 +371,66 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			ip.unusedSize += size
 			ip.unusedBlobs++
 		}
+		if !blob.IsCompressed() {
+			ip.uncompressed = true
+		}
 		// update indexPack
 		indexPack[blob.PackID] = ip
 	}
 
-	Verbosef("collecting packs for deletion and repacking\n")
+	// if duplicate blobs exist, those will be set to either "used" or "unused":
+	// - mark only one occurence of duplicate blobs as used
+	// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
+	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
+	if len(duplicateBlobs) > 0 {
+		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
+		for blob := range idx.Each(ctx) {
+			bh := blob.BlobHandle
+			count, isDuplicate := duplicateBlobs[bh]
+			if !isDuplicate {
+				continue
+			}
+
+			ip := indexPack[blob.PackID]
+			size := uint64(blob.Length)
+			switch {
+			case count == 0:
+				// used duplicate exists ->  mark as unused
+				ip.unusedSize += size
+				ip.unusedBlobs++
+			case ip.usedBlobs > 0, count == 1:
+				// other used blobs in pack or "last" occurency ->  mark as used
+				ip.usedSize += size
+				ip.usedBlobs++
+				// let other occurences be marked as unused
+				duplicateBlobs[bh] = 0
+			default:
+				// mark as unused and decrease counter
+				ip.unusedSize += size
+				ip.unusedBlobs++
+				duplicateBlobs[bh] = count - 1
+			}
+			// update indexPack
+			indexPack[blob.PackID] = ip
+		}
+	}
+
+	return keepBlobs, indexPack, nil
+}
+
+func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats) (prunePlan, error) {
 	removePacksFirst := restic.NewIDSet()
 	removePacks := restic.NewIDSet()
 	repackPacks := restic.NewIDSet()
 
 	var repackCandidates []packInfoWithID
-	repackAllPacksWithDuplicates := true
-
-	keep := func(p packInfo) {
-		stats.packs.keep++
-		if p.duplicateBlobs > 0 {
-			repackAllPacksWithDuplicates = false
-		}
+	var repackSmallCandidates []packInfoWithID
+	repoVersion := repo.Config().Version
+	// only repack very small files by default
+	targetPackSize := repo.PackSize() / 25
+	if opts.RepackSmall {
+		// consider files with at least 80% of the target size as large enough
+		targetPackSize = repo.PackSize() / 5 * 4
 	}
 
 	// loop over all packs and decide what to do
@@ -319,8 +445,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			return nil
 		}
 
-		if p.unusedSize+p.usedSize != uint64(packSize) &&
-			!(p.usedBlobs == 0 && p.duplicateBlobs == 0) {
+		if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
 			// Pack size does not fit and pack is needed => error
 			// If the pack is not needed, this is no error, the pack can
 			// and will be simply removed, see below.
@@ -331,7 +456,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 		// statistics
 		switch {
-		case p.usedBlobs == 0 && p.duplicateBlobs == 0:
+		case p.usedBlobs == 0:
 			stats.packs.unused++
 		case p.unusedBlobs == 0:
 			stats.packs.used++
@@ -339,9 +464,18 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			stats.packs.partlyUsed++
 		}
 
+		mustCompress := false
+		if repoVersion >= 2 {
+			// repo v2: always repack tree blobs if uncompressed
+			// compress data blobs if requested
+			mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
+		}
+		// use a flag that pack must be compressed
+		p.uncompressed = mustCompress
+
 		// decide what to do
 		switch {
-		case p.usedBlobs == 0 && p.duplicateBlobs == 0:
+		case p.usedBlobs == 0:
 			// All blobs in pack are no longer used => remove pack!
 			removePacks.Insert(id)
 			stats.blobs.remove += p.unusedBlobs
@@ -349,11 +483,15 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 		case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
 			// if this is a data pack and --repack-cacheable-only is set => keep pack!
-			keep(p)
+			stats.packs.keep++
 
-		case p.unusedBlobs == 0 && p.duplicateBlobs == 0 && p.tpe != restic.InvalidBlob:
-			// All blobs in pack are used and not duplicates/mixed => keep pack!
-			keep(p)
+		case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
+			if packSize >= int64(targetPackSize) {
+				// All blobs in pack are used and not mixed => keep pack!
+				stats.packs.keep++
+			} else {
+				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p})
+			}
 
 		default:
 			// all other packs are candidates for repacking
@@ -366,7 +504,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	})
 	bar.Done()
 	if err != nil {
-		return err
+		return prunePlan{}, err
 	}
 
 	// At this point indexPacks contains only missing packs!
@@ -374,7 +512,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	// missing packs that are not needed can be ignored
 	ignorePacks := restic.NewIDSet()
 	for id, p := range indexPack {
-		if p.usedBlobs == 0 && p.duplicateBlobs == 0 {
+		if p.usedBlobs == 0 {
 			ignorePacks.Insert(id)
 			stats.blobs.remove += p.unusedBlobs
 			stats.size.remove += p.unusedSize
@@ -387,7 +525,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		for id := range indexPack {
 			Warnf("  %v\n", id)
 		}
-		return errorPacksMissing
+		return prunePlan{}, errorPacksMissing
 	}
 	if len(ignorePacks) != 0 {
 		Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
@@ -396,89 +534,101 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		}
 	}
 
-	// calculate limit for number of unused bytes in the repo after repacking
-	maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
+	if len(repackSmallCandidates) < 10 {
+		// too few small files to be worth the trouble, this also prevents endlessly repacking
+		// if there is just a single pack file below the target size
+		stats.packs.keep += uint(len(repackSmallCandidates))
+	} else {
+		repackCandidates = append(repackCandidates, repackSmallCandidates...)
+	}
 
 	// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
 	// This is equivalent to sorting by unused / total space.
 	// Instead of unused[i] / used[i] > unused[j] / used[j] we use
 	// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
-	// Morover duplicates and packs containing trees are sorted to the beginning
+	// Moreover packs containing trees and too small packs are sorted to the beginning
 	sort.Slice(repackCandidates, func(i, j int) bool {
 		pi := repackCandidates[i].packInfo
 		pj := repackCandidates[j].packInfo
 		switch {
-		case pi.duplicateBlobs > 0 && pj.duplicateBlobs == 0:
-			return true
-		case pj.duplicateBlobs > 0 && pi.duplicateBlobs == 0:
-			return false
 		case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
 			return true
 		case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
 			return false
+		case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
+			return true
+		case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
+			return false
 		}
 		return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
 	})
 
 	repack := func(id restic.ID, p packInfo) {
 		repackPacks.Insert(id)
-		stats.blobs.repack += p.unusedBlobs + p.duplicateBlobs + p.usedBlobs
+		stats.blobs.repack += p.unusedBlobs + p.usedBlobs
 		stats.size.repack += p.unusedSize + p.usedSize
 		stats.blobs.repackrm += p.unusedBlobs
 		stats.size.repackrm += p.unusedSize
 	}
 
+	// calculate limit for number of unused bytes in the repo after repacking
+	maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
+
 	for _, p := range repackCandidates {
 		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
-
-		reachedRepackSize := false
-		if opts.MaxRepackBytes > 0 {
-			reachedRepackSize = stats.size.repack+p.unusedSize+p.usedSize > opts.MaxRepackBytes
-		}
+		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
+		packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
 
 		switch {
 		case reachedRepackSize:
-			keep(p.packInfo)
+			stats.packs.keep++
 
-		case p.duplicateBlobs > 0, p.tpe != restic.DataBlob:
-			// repacking duplicates/non-data is only limited by repackSize
+		case p.tpe != restic.DataBlob, p.uncompressed:
+			// repacking non-data packs / uncompressed-trees is only limited by repackSize
 			repack(p.ID, p.packInfo)
 
-		case reachedUnusedSizeAfter:
+		case reachedUnusedSizeAfter && packIsLargeEnough:
 			// for all other packs stop repacking if tolerated unused size is reached.
-			keep(p.packInfo)
+			stats.packs.keep++
 
 		default:
 			repack(p.ID, p.packInfo)
 		}
 	}
 
-	// if all duplicates are repacked, print out correct statistics
-	if repackAllPacksWithDuplicates {
-		stats.blobs.repackrm += stats.blobs.duplicate
-		stats.size.repackrm += stats.size.duplicate
-	}
+	stats.packs.unref = uint(len(removePacksFirst))
+	stats.packs.repack = uint(len(repackPacks))
+	stats.packs.remove = uint(len(removePacks))
 
-	Verboseff("\nused:        %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
+	return prunePlan{removePacksFirst: removePacksFirst,
+		removePacks: removePacks,
+		repackPacks: repackPacks,
+		ignorePacks: ignorePacks,
+	}, nil
+}
+
+// printPruneStats prints out the statistics
+func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
+	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
 	if stats.blobs.duplicate > 0 {
-		Verboseff("duplicates:  %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
+		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
 	}
-	Verboseff("unused:      %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
+	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
 	if stats.size.unref > 0 {
-		Verboseff("unreferenced:                   %s\n", formatBytes(stats.size.unref))
+		Verboseff("unreferenced:                    %s\n", formatBytes(stats.size.unref))
 	}
 	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
 	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
 	unusedSize := stats.size.duplicate + stats.size.unused
-	Verboseff("total:       %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
+	Verboseff("total:        %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
 	Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
 
-	Verbosef("\nto repack:   %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
-	Verbosef("this removes %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
-	Verbosef("to delete:   %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
+	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
+	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
+	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
 	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
-	Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
-	Verbosef("remaining:   %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
+	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
+	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
 	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
 	Verbosef("unused size after prune: %s (%s of remaining size)\n",
 		formatBytes(unusedAfter), formatPercent(unusedAfter, totalSize-totalPruneSize))
@@ -487,74 +637,110 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
 	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
 
-	Verboseff("to keep:   %10d packs\n", stats.packs.keep)
-	Verboseff("to repack: %10d packs\n", len(repackPacks))
-	Verboseff("to delete: %10d packs\n", len(removePacks))
-	if len(removePacksFirst) > 0 {
-		Verboseff("to delete: %10d unreferenced packs\n\n", len(removePacksFirst))
+	Verboseff("to keep:      %10d packs\n", stats.packs.keep)
+	Verboseff("to repack:    %10d packs\n", stats.packs.repack)
+	Verboseff("to delete:    %10d packs\n", stats.packs.remove)
+	if stats.packs.unref > 0 {
+		Verboseff("to delete:    %10d unreferenced packs\n\n", stats.packs.unref)
 	}
+	return nil
+}
+
+// doPrune does the actual pruning:
+// - remove unreferenced packs first
+// - repack given pack files while keeping the given blobs
+// - rebuild the index while ignoring all files that will be deleted
+// - delete the files
+// plan.removePacks and plan.ignorePacks are modified in this function.
+func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
+	ctx := gopts.ctx
 
 	if opts.DryRun {
 		if !gopts.JSON && gopts.verbosity >= 2 {
-			if len(removePacksFirst) > 0 {
-				Printf("Would have removed the following unreferenced packs:\n%v\n\n", removePacksFirst)
+			if len(plan.removePacksFirst) > 0 {
+				Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
 			}
-			Printf("Would have repacked and removed the following packs:\n%v\n\n", repackPacks)
-			Printf("Would have removed the following no longer used packs:\n%v\n\n", removePacks)
+			Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
+			Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
 		}
 		// Always quit here if DryRun was set!
 		return nil
 	}
 
 	// unreferenced packs can be safely deleted first
-	if len(removePacksFirst) != 0 {
+	if len(plan.removePacksFirst) != 0 {
 		Verbosef("deleting unreferenced packs\n")
-		DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile)
+		DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile)
 	}
 
-	if len(repackPacks) != 0 {
+	if len(plan.repackPacks) != 0 {
 		Verbosef("repacking packs\n")
-		bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked")
-		_, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar)
+		bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked")
+		_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
 		bar.Done()
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
 
 		// Also remove repacked packs
-		removePacks.Merge(repackPacks)
+		plan.removePacks.Merge(plan.repackPacks)
+
+		if len(plan.keepBlobs) != 0 {
+			Warnf("%v was not repacked\n\n"+
+				"Integrity check failed.\n"+
+				"Please report this error (along with the output of the 'prune' run) at\n"+
+				"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
+			return errors.Fatal("internal error: blobs were not repacked")
+		}
 	}
 
-	if len(ignorePacks) == 0 {
-		ignorePacks = removePacks
+	if len(plan.ignorePacks) == 0 {
+		plan.ignorePacks = plan.removePacks
 	} else {
-		ignorePacks.Merge(removePacks)
+		plan.ignorePacks.Merge(plan.removePacks)
 	}
 
-	if len(ignorePacks) != 0 {
-		err = rebuildIndexFiles(gopts, repo, ignorePacks, nil)
+	if opts.unsafeRecovery {
+		Verbosef("deleting index files\n")
+		indexFiles := repo.Index().(*repository.MasterIndex).IDs()
+		err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+	} else if len(plan.ignorePacks) != 0 {
+		err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil)
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
 	}
 
-	if len(removePacks) != 0 {
-		Verbosef("removing %d old packs\n", len(removePacks))
-		DeleteFiles(gopts, repo, removePacks, restic.PackFile)
+	if len(plan.removePacks) != 0 {
+		Verbosef("removing %d old packs\n", len(plan.removePacks))
+		DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile)
+	}
+
+	if opts.unsafeRecovery {
+		_, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
 	}
 
 	Verbosef("done\n")
 	return nil
 }
 
-func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
+func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
 	Verbosef("rebuilding index\n")
 
-	idx := (repo.Index()).(*repository.MasterIndex)
-	packcount := uint64(len(idx.Packs(removePacks)))
-	bar := newProgressMax(!gopts.Quiet, packcount, "packs processed")
-	obsoleteIndexes, err := idx.Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
+	bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
+	obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
 	bar.Done()
+	return obsoleteIndexes, err
+}
+
+func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
+	obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete)
 	if err != nil {
 		return err
 	}
@@ -568,17 +754,18 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r
 
 	var snapshotTrees restic.IDs
 	Verbosef("loading all snapshots...\n")
-	err = restic.ForAllSnapshots(gopts.ctx, repo, ignoreSnapshots,
+	err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots,
 		func(id restic.ID, sn *restic.Snapshot, err error) error {
-			debug.Log("add snapshot %v (tree %v, error %v)", id, *sn.Tree, err)
 			if err != nil {
+				debug.Log("failed to load snapshot %v (error %v)", id, err)
 				return err
 			}
+			debug.Log("add snapshot %v (tree %v)", id, *sn.Tree)
 			snapshotTrees = append(snapshotTrees, *sn.Tree)
 			return nil
 		})
 	if err != nil {
-		return nil, err
+		return nil, errors.Fatalf("failed loading snapshot: %v", err)
 	}
 
 	Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
@@ -591,7 +778,7 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r
 	err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
 	if err != nil {
 		if repo.Backend().IsNotExist(err) {
-			return nil, errors.Fatal("unable to load a tree from the repo: " + err.Error())
+			return nil, errors.Fatal("unable to load a tree from the repository: " + err.Error())
 		}
 
 		return nil, err

cmd/restic/cmd_rebuild_index.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 
@@ -73,11 +74,31 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		}
 	} else {
 		Verbosef("loading indexes...\n")
-		err := repo.LoadIndex(gopts.ctx)
+		mi := repository.NewMasterIndex()
+		err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+			if err != nil {
+				Warnf("removing invalid index %v: %v\n", id, err)
+				obsoleteIndexes = append(obsoleteIndexes, id)
+				return nil
+			}
+
+			mi.Insert(idx)
+			return nil
+		})
 		if err != nil {
 			return err
 		}
-		packSizeFromIndex = repo.Index().PackSize(ctx, false)
+
+		err = mi.MergeFinalIndexes()
+		if err != nil {
+			return err
+		}
+
+		err = repo.SetIndex(mi)
+		if err != nil {
+			return err
+		}
+		packSizeFromIndex = pack.Size(ctx, repo.Index(), false)
 	}
 
 	Verbosef("getting pack files to read...\n")

cmd/restic/cmd_recover.go

@@ -1,21 +1,24 @@
 package main
 
 import (
+	"context"
 	"os"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
 )
 
 var cmdRecover = &cobra.Command{
 	Use:   "recover [flags]",
-	Short: "Recover data from the repository",
+	Short: "Recover data from the repository not referenced by snapshots",
 	Long: `
 The "recover" command builds a new snapshot from all directories it can find in
-the raw data of the repository. It can be used if, for example, a snapshot has
-been removed by accident with "forget".
+the raw data of the repository which are not referenced in an existing snapshot.
+It can be used if, for example, a snapshot has been removed by accident with "forget".
 
 EXIT STATUS
 ===========
@@ -49,6 +52,11 @@ func runRecover(gopts GlobalOptions) error {
 		return err
 	}
 
+	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	Verbosef("load index files\n")
 	if err = repo.LoadIndex(gopts.ctx); err != nil {
 		return err
@@ -59,53 +67,54 @@ func runRecover(gopts GlobalOptions) error {
 	trees := make(map[restic.ID]bool)
 
 	for blob := range repo.Index().Each(gopts.ctx) {
-		if blob.Blob.Type != restic.TreeBlob {
-			continue
+		if blob.Type == restic.TreeBlob {
+			trees[blob.Blob.ID] = false
 		}
-		trees[blob.Blob.ID] = false
 	}
 
-	cur := 0
-	max := len(trees)
-	Verbosef("load %d trees\n\n", len(trees))
-
+	Verbosef("load %d trees\n", len(trees))
+	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
-		cur++
-		Verbosef("\rtree (%v/%v)", cur, max)
-
-		if !trees[id] {
-			trees[id] = false
-		}
-
-		tree, err := repo.LoadTree(gopts.ctx, id)
+		tree, err := restic.LoadTree(gopts.ctx, repo, id)
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
 			continue
 		}
 
 		for _, node := range tree.Nodes {
-			if node.Type != "dir" || node.Subtree == nil {
-				continue
+			if node.Type == "dir" && node.Subtree != nil {
+				trees[*node.Subtree] = true
 			}
-
-			subtree := *node.Subtree
-			trees[subtree] = true
 		}
+		bar.Add(1)
 	}
-	Verbosef("\ndone\n")
+	bar.Done()
+
+	Verbosef("load snapshots\n")
+	err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+		trees[*sn.Tree] = true
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	Verbosef("done\n")
 
 	roots := restic.NewIDSet()
 	for id, seen := range trees {
-		if seen {
-			continue
+		if !seen {
+			Verboseff("found root tree %v\n", id.Str())
+			roots.Insert(id)
 		}
+	}
+	Printf("\nfound %d unreferenced roots\n", len(roots))
 
-		roots.Insert(id)
+	if len(roots) == 0 {
+		Verbosef("no snapshot to write.\n")
+		return nil
 	}
 
-	Verbosef("found %d roots\n", len(roots))
-
-	tree := restic.NewTree()
+	tree := restic.NewTree(len(roots))
 	for id := range roots {
 		var subtreeID = id
 		node := restic.Node{
@@ -117,35 +126,51 @@ func runRecover(gopts GlobalOptions) error {
 			ModTime:    time.Now(),
 			ChangeTime: time.Now(),
 		}
-		err = tree.Insert(&node)
+		err := tree.Insert(&node)
 		if err != nil {
 			return err
 		}
 	}
 
-	treeID, err := repo.SaveTree(gopts.ctx, tree)
+	wg, ctx := errgroup.WithContext(gopts.ctx)
+	repo.StartPackUploader(ctx, wg)
+
+	var treeID restic.ID
+	wg.Go(func() error {
+		var err error
+		treeID, err = restic.SaveTree(ctx, repo, tree)
+		if err != nil {
+			return errors.Fatalf("unable to save new tree to the repository: %v", err)
+		}
+
+		err = repo.Flush(ctx)
+		if err != nil {
+			return errors.Fatalf("unable to save blobs to the repository: %v", err)
+		}
+		return nil
+	})
+	err = wg.Wait()
 	if err != nil {
-		return errors.Fatalf("unable to save new tree to the repo: %v", err)
+		return err
 	}
 
-	err = repo.Flush(gopts.ctx)
-	if err != nil {
-		return errors.Fatalf("unable to save blobs to the repo: %v", err)
-	}
+	return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
 
-	sn, err := restic.NewSnapshot([]string{"/recover"}, []string{}, hostname, time.Now())
+}
+
+func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
+	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
-	sn.Tree = &treeID
+	sn.Tree = tree
 
-	id, err := repo.SaveJSONUnpacked(gopts.ctx, restic.SnapshotFile, sn)
+	id, err := restic.SaveSnapshot(ctx, repo, sn)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
 	Printf("saved new snapshot %v\n", id.Str())
-
 	return nil
 }