Add version for 0.14.0

Fix typo in the environment variable name for `--from-password-file`

.gitattributes

@@ -0,0 +1,2 @@
+# Workaround for https://github.com/golang/go/issues/52268.
+**/testdata/fuzz/*/* eol=lf

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ on:
   pull_request:
 
 env:
-  latest_go: "1.18.x"
+  latest_go: "1.19.x"
   GO111MODULE: on
 
 jobs:
@@ -19,24 +19,30 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.18.x
+            go: 1.19.x
             os: windows-latest
             install_verb: install
 
           - job_name: macOS
-            go: 1.18.x
+            go: 1.19.x
             os: macOS-latest
             test_fuse: false
             install_verb: install
 
           - job_name: Linux
-            go: 1.18.x
+            go: 1.19.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
             install_verb: install
 
+          - job_name: Linux
+            go: 1.18.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: install
+
           - job_name: Linux
             go: 1.17.x
             os: ubuntu-latest
@@ -55,12 +61,6 @@ jobs:
             test_fuse: true
             install_verb: get
 
-          - job_name: Linux
-            go: 1.14.x
-            os: ubuntu-latest
-            test_fuse: true
-            install_verb: get
-
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
 
@@ -266,7 +266,7 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.45
+          version: v1.48
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           only-new-issues: true
           args: --verbose --timeout 5m

.golangci.yml

@@ -24,7 +24,7 @@ linters:
     - govet
 
     # make sure names and comments are used according to the conventions
-    - golint
+    - revive
 
     # detect when assignments to existing variables are not used
     - ineffassign
@@ -51,7 +51,7 @@ issues:
 
   # list of things to not warn about
   exclude:
-    # golint: do not warn about missing comments for exported stuff
-    - exported (function|method|var|type|const) `.*` should have comment or be unexported
-    # golint: ignore constants in all caps
+    # revive: do not warn about missing comments for exported stuff
+    - exported (function|method|var|type|const) .* should have comment or be unexported
+    # revive: ignore constants in all caps
     - don't use ALL_CAPS in Go names; use CamelCase

CHANGELOG.md

@@ -1,3 +1,453 @@
+Changelog for restic 0.14.0 (2022-08-25)
+=======================================
+
+The following sections list the changes in restic 0.14.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2248: Support `self-update` on Windows
+ * Fix #3428: List snapshots in backend at most once to resolve snapshot IDs
+ * Fix #3432: Fix rare 'not found in repository' error for `copy` command
+ * Fix #3685: The `diff` command incorrectly listed some files as added
+ * Fix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows
+ * Fix #3720: Directory sync errors for repositories accessed via SMB
+ * Fix #3736: The `stats` command miscalculated restore size for multiple snapshots
+ * Fix #3861: Yield error on invalid policy to `forget`
+ * Fix #3716: Print "wrong password" to stderr instead of stdout
+ * Fix #3772: Correctly rebuild index for legacy repositories
+ * Fix #3776: Limit number of key files tested while opening a repository
+ * Chg #1842: Support debug log creation in release builds
+ * Chg #3295: Deprecate `check --check-unused` and add further checks
+ * Chg #3680: Update dependencies and require Go 1.15 or newer
+ * Chg #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
+ * Enh #1153: Support pruning even when the disk is full
+ * Enh #21: Add compression support
+ * Enh #2162: Adaptive IO concurrency based on backend connections
+ * Enh #2291: Allow pack size customization
+ * Enh #2295: Allow use of SAS token to authenticate to Azure
+ * Enh #2696: Improve backup speed with many small files
+ * Enh #2907: Make snapshot directory structure of `mount` command customizable
+ * Enh #3114: Optimize handling of duplicate blobs in `prune`
+ * Enh #3465: Improve handling of temporary files on Windows
+ * Enh #3709: Validate exclude patterns before backing up
+ * Enh #3837: Improve SFTP repository initialization over slow links
+ * Enh #2351: Use config file permissions to control file group access
+ * Enh #3475: Allow limiting IO concurrency for local and SFTP backend
+ * Enh #3484: Stream data in `check` and `prune` commands
+ * Enh #2923: Improve speed of `copy` command
+ * Enh #3729: Display full IDs in `check` warnings
+ * Enh #3773: Optimize memory usage for directories with many files
+ * Enh #3819: Validate include/exclude patterns before restoring
+
+Details
+-------
+
+ * Bugfix #2248: Support `self-update` on Windows
+
+   Restic `self-update` would fail in situations where the operating system locks running
+   binaries, including Windows. The new behavior works around this by renaming the running file
+   and swapping the updated file in place.
+
+   https://github.com/restic/restic/issues/2248
+   https://github.com/restic/restic/pull/3675
+
+ * Bugfix #3428: List snapshots in backend at most once to resolve snapshot IDs
+
+   Many commands support specifying a list of snapshot IDs which are then used to determine the
+   snapshots to be processed by the command. To resolve snapshot IDs or `latest`, and check that
+   these exist, restic previously listed all snapshots stored in the repository. Depending on
+   the backend this could be a slow and/or expensive operation.
+
+   Restic now lists the snapshots only once and remembers the result in order to resolve all
+   further snapshot IDs swiftly.
+
+   https://github.com/restic/restic/issues/3428
+   https://github.com/restic/restic/pull/3570
+   https://github.com/restic/restic/pull/3395
+
+ * Bugfix #3432: Fix rare 'not found in repository' error for `copy` command
+
+   In rare cases `copy` (and other commands) would report that `LoadTree(...)` returned an `id
+   [...] not found in repository` error. This could be caused by a backup or copy command running
+   concurrently. The error was only temporary; running the failed restic command a second time as
+   a workaround did resolve the error.
+
+   This issue has now been fixed by correcting the order in which restic reads data from the
+   repository. It is now guaranteed that restic only loads snapshots for which all necessary data
+   is already available.
+
+   https://github.com/restic/restic/issues/3432
+   https://github.com/restic/restic/pull/3570
+
+ * Bugfix #3685: The `diff` command incorrectly listed some files as added
+
+   There was a bug in the `diff` command, causing it to always show files in a removed directory as
+   added. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3685
+   https://github.com/restic/restic/pull/3686
+
+ * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows
+
+   In #3602 a fix was introduced to address the problem of `rclone` prematurely exiting when
+   Ctrl+C is pressed on Windows. The solution was to create the subprocess with its console
+   detached from the restic console.
+
+   However, this solution failed when using `rclone` installed by Scoop or using `sftp` with a
+   passphrase-protected private key. We've now fixed this by using a different approach to
+   prevent Ctrl-C from passing down too early.
+
+   https://github.com/restic/restic/issues/3681
+   https://github.com/restic/restic/issues/3692
+   https://github.com/restic/restic/pull/3696
+
+ * Bugfix #3720: Directory sync errors for repositories accessed via SMB
+
+   On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in restic failing to
+   save the lock file, yielding the following errors:
+
+   Save(<lock/071fe833f0>) returned error, retrying after 552.330144ms: sync /repo/locks:
+   no such file or directory Save(<lock/bf789d7343>) returned error, retrying after
+   552.330144ms: sync /repo/locks: invalid argument
+
+   This has now been fixed by ignoring the relevant error codes.
+
+   https://github.com/restic/restic/issues/3720
+   https://github.com/restic/restic/issues/3751
+   https://github.com/restic/restic/pull/3752
+
+ * Bugfix #3736: The `stats` command miscalculated restore size for multiple snapshots
+
+   Since restic 0.10.0 the restore size calculated by the `stats` command for multiple snapshots
+   was too low. The hardlink detection was accidentally applied across multiple snapshots and
+   thus ignored many files. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3736
+   https://github.com/restic/restic/pull/3740
+
+ * Bugfix #3861: Yield error on invalid policy to `forget`
+
+   The `forget` command previously silently ignored invalid/unsupported units in the duration
+   options, such as e.g. `--keep-within-daily 2w`.
+
+   Specifying an invalid/unsupported duration unit now results in an error.
+
+   https://github.com/restic/restic/issues/3861
+   https://github.com/restic/restic/pull/3862
+
+ * Bugfix #3716: Print "wrong password" to stderr instead of stdout
+
+   If an invalid password was entered, the error message was printed on stdout and not on stderr as
+   intended. This has now been fixed.
+
+   https://github.com/restic/restic/pull/3716
+   https://forum.restic.net/t/4965
+
+ * Bugfix #3772: Correctly rebuild index for legacy repositories
+
+   After running `rebuild-index` on a legacy repository containing mixed pack files (that is,
+   pack files which store both metadata and file data), `check` printed warnings like `pack
+   12345678 contained in several indexes: ...`. This warning was not critical, but has now
+   nonetheless been fixed by properly handling mixed pack files while rebuilding the index.
+
+   Running `prune` for such legacy repositories will also fix the warning by reorganizing the
+   pack files which caused it.
+
+   https://github.com/restic/restic/pull/3772
+   https://github.com/restic/restic/pull/3884
+   https://forum.restic.net/t/5044/13
+
+ * Bugfix #3776: Limit number of key files tested while opening a repository
+
+   Previously, restic tested the password against every key in the repository when opening a
+   repository. The more keys there were in the repository, the slower this operation became.
+
+   Restic now tests the password against up to 20 key files in the repository. Alternatively, you
+   can use the `--key-hint=<key ID>` option to specify a specific key file to use instead.
+
+   https://github.com/restic/restic/pull/3776
+
+ * Change #1842: Support debug log creation in release builds
+
+   Creating a debug log was only possible in debug builds which required users to manually build
+   restic. We changed the release builds to allow creating debug logs by simply setting the
+   environment variable `DEBUG_LOG=logname.log`.
+
+   https://github.com/restic/restic/issues/1842
+   https://github.com/restic/restic/pull/3826
+
+ * Change #3295: Deprecate `check --check-unused` and add further checks
+
+   Since restic 0.12.0, it is expected to still have unused blobs after running `prune`. This made
+   the `--check-unused` option of the `check` command rather useless and tended to confuse
+   users. This option has been deprecated and is now ignored.
+
+   The `check` command now also warns if a repository is using either the legacy S3 layout or mixed
+   pack files with both tree and data blobs. The latter is known to cause performance problems.
+
+   https://github.com/restic/restic/issues/3295
+   https://github.com/restic/restic/pull/3730
+
+ * Change #3680: Update dependencies and require Go 1.15 or newer
+
+   We've updated most dependencies. Since some libraries require newer language features we're
+   dropping support for Go 1.14, which means that restic now requires at least Go 1.15 to build.
+
+   https://github.com/restic/restic/issues/3680
+   https://github.com/restic/restic/issues/3883
+
+ * Change #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
+
+   The `init` and `copy` commands can read data from another repository. However, confusingly
+   `--repo2` referred to the repository *from* which the `init` command copies parameters, but
+   for the `copy` command `--repo2` referred to the copy *destination*.
+
+   We've introduced a new option, `--from-repo`, which always refers to the source repository
+   for both commands. The old parameter names have been deprecated but still work. To create a new
+   repository and copy all snapshots to it, the commands are now as follows:
+
+   ``` restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo
+   --copy-chunker-params restic -r /srv/restic-repo-copy copy --from-repo
+   /srv/restic-repo ```
+
+   https://github.com/restic/restic/pull/3742
+   https://forum.restic.net/t/5017
+
+ * Enhancement #1153: Support pruning even when the disk is full
+
+   When running out of disk space it was no longer possible to add or remove data from a repository.
+   To help with recovering from such a deadlock, the prune command now supports an
+   `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to
+   read the documentation first!
+
+   https://github.com/restic/restic/issues/1153
+   https://github.com/restic/restic/pull/3481
+
+ * Enhancement #21: Add compression support
+
+   We've added compression support to the restic repository format. To create a repository using
+   the new format run `init --repository-version 2`. Please note that the repository cannot be
+   read by restic versions prior to 0.14.0.
+
+   You can configure whether data is compressed with the option `--compression`. It can be set to
+   `auto` (the default, which will compress very fast), `max` (which will trade backup speed and
+   CPU usage for better compression), or `off` (which disables compression). Each setting is
+   only applied for the current run of restic and does *not* apply to future runs. The option can
+   also be set via the environment variable `RESTIC_COMPRESSION`.
+
+   To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the documentation
+   for more details. The migration checks the repository integrity and upgrades the repository
+   format, but will not change any data. Afterwards, prune will rewrite the metadata to make use of
+   compression.
+
+   As an alternative you can use the `copy` command to migrate snapshots; First create a new
+   repository using `init --repository-version 2 --copy-chunker-params --repo2
+   path/to/old/repo`, and then use the `copy` command to copy all snapshots to the new
+   repository.
+
+   https://github.com/restic/restic/issues/21
+   https://github.com/restic/restic/issues/3779
+   https://github.com/restic/restic/pull/3666
+   https://github.com/restic/restic/pull/3704
+   https://github.com/restic/restic/pull/3733
+
+ * Enhancement #2162: Adaptive IO concurrency based on backend connections
+
+   Many commands used hard-coded limits for the number of concurrent operations. This prevented
+   speed improvements by increasing the number of connections used by a backend.
+
+   These limits have now been replaced by using the configured number of backend connections
+   instead, which can be controlled using the `-o <backend-name>.connections=5` option.
+   Commands will then automatically scale their parallelism accordingly.
+
+   To limit the number of CPU cores used by restic, you can set the environment variable
+   `GOMAXPROCS` accordingly. For example to use a single CPU core, use `GOMAXPROCS=1`.
+
+   https://github.com/restic/restic/issues/2162
+   https://github.com/restic/restic/issues/1467
+   https://github.com/restic/restic/pull/3611
+
+ * Enhancement #2291: Allow pack size customization
+
+   Restic now uses a target pack size of 16 MiB by default. This can be customized using the
+   `--pack-size size` option. Supported pack sizes range between 4 and 128 MiB.
+
+   It is possible to migrate an existing repository to _larger_ pack files using `prune
+   --repack-small`. This will rewrite every pack file which is significantly smaller than the
+   target size.
+
+   https://github.com/restic/restic/issues/2291
+   https://github.com/restic/restic/pull/3731
+
+ * Enhancement #2295: Allow use of SAS token to authenticate to Azure
+
+   Previously restic only supported AccountKeys to authenticate to Azure storage accounts,
+   which necessitates giving a significant amount of access.
+
+   We added support for Azure SAS tokens which are a more fine-grained and time-limited manner of
+   granting access. Set the `AZURE_ACCOUNT_NAME` and `AZURE_ACCOUNT_SAS` environment
+   variables to use a SAS token for authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it
+   will take precedence.
+
+   https://github.com/restic/restic/issues/2295
+   https://github.com/restic/restic/pull/3661
+
+ * Enhancement #2696: Improve backup speed with many small files
+
+   We have restructured the backup pipeline to continue reading files while all upload
+   connections are busy. This allows the backup to already prepare the next data file such that the
+   upload can continue as soon as a connection becomes available. This can especially improve the
+   backup performance for high latency backends.
+
+   The upload concurrency is now controlled using the `-o <backend-name>.connections=5`
+   option.
+
+   https://github.com/restic/restic/issues/2696
+   https://github.com/restic/restic/pull/3489
+
+ * Enhancement #2907: Make snapshot directory structure of `mount` command customizable
+
+   We've added the possibility to customize the snapshot directory structure of the `mount`
+   command using templates passed to the `--snapshot-template` option. The formatting of
+   snapshots' timestamps is now controlled using `--time-template` and supports
+   subdirectories to for example group snapshots by year. Please see `restic help mount` for
+   further details.
+
+   Characters in tag names which are not allowed in a filename are replaced by underscores `_`. For
+   example a tag `foo/bar` will result in a directory name of `foo_bar`.
+
+   https://github.com/restic/restic/issues/2907
+   https://github.com/restic/restic/pull/2913
+   https://github.com/restic/restic/pull/3691
+
+ * Enhancement #3114: Optimize handling of duplicate blobs in `prune`
+
+   Restic `prune` always used to repack all data files containing duplicate blobs. This
+   effectively removed all duplicates during prune. However, as a consequence all these data
+   files were repacked even if the unused repository space threshold could be reached with less
+   work.
+
+   This is now changed and `prune` works nice and fast even when there are lots of duplicate blobs.
+
+   https://github.com/restic/restic/issues/3114
+   https://github.com/restic/restic/pull/3290
+
+ * Enhancement #3465: Improve handling of temporary files on Windows
+
+   In some cases restic failed to delete temporary files, causing the current command to fail.
+   This has now been fixed by ensuring that Windows automatically deletes the file. In addition,
+   temporary files are only written to disk when necessary, reducing disk writes.
+
+   https://github.com/restic/restic/issues/3465
+   https://github.com/restic/restic/issues/1551
+   https://github.com/restic/restic/pull/3610
+
+ * Enhancement #3709: Validate exclude patterns before backing up
+
+   Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or
+   `--iexclude-file` previously weren't validated. As a consequence, invalid patterns
+   resulted in files that were meant to be excluded being backed up.
+
+   Restic now validates all patterns before running the backup and aborts with a fatal error if an
+   invalid pattern is detected.
+
+   https://github.com/restic/restic/issues/3709
+   https://github.com/restic/restic/pull/3734
+
+ * Enhancement #3837: Improve SFTP repository initialization over slow links
+
+   The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the
+   backend concurrently. This reduces the waiting times when creating a repository over a very
+   slow connection.
+
+   https://github.com/restic/restic/issues/3837
+   https://github.com/restic/restic/pull/3840
+
+ * Enhancement #2351: Use config file permissions to control file group access
+
+   Previously files in a local/SFTP repository would always end up with very restrictive access
+   permissions, allowing access only to the owner. This prevented a number of valid use-cases
+   involving groups and ACLs.
+
+   We now use the permissions of the config file in the repository to decide whether group access
+   should be given to newly created repository files or not. We arrange for repository files to be
+   created group readable exactly when the repository config file is group readable.
+
+   To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config
+   file can be used. For repositories that should be writable by group members a tad more setup is
+   required, see the docs.
+
+   Posix ACLs can also be used now that the group permissions being forced to zero no longer masks
+   the effect of ACL entries.
+
+   https://github.com/restic/restic/issues/2351
+   https://github.com/restic/restic/pull/3419
+   https://forum.restic.net/t/1391
+
+ * Enhancement #3475: Allow limiting IO concurrency for local and SFTP backend
+
+   Restic did not support limiting the IO concurrency / number of connections for accessing
+   repositories stored using the local or SFTP backends. The number of connections is now limited
+   as for other backends, and can be configured via the the `-o local.connections=2` and `-o
+   sftp.connections=5` options. This ensures that restic does not overwhelm the backend with
+   concurrent IO operations.
+
+   https://github.com/restic/restic/pull/3475
+
+ * Enhancement #3484: Stream data in `check` and `prune` commands
+
+   The commands `check --read-data` and `prune` previously downloaded data files into
+   temporary files which could end up being written to disk. This could cause a large amount of data
+   being written to disk.
+
+   The pack files are now instead streamed, which removes the need for temporary files. Please
+   note that *uploads* during `backup` and `prune` still require temporary files.
+
+   https://github.com/restic/restic/issues/3710
+   https://github.com/restic/restic/pull/3484
+   https://github.com/restic/restic/pull/3717
+
+ * Enhancement #2923: Improve speed of `copy` command
+
+   The `copy` command could require a long time to copy snapshots for non-local backends. This has
+   been improved to provide a throughput comparable to the `restore` command.
+
+   Additionally, `copy` now displays a progress bar.
+
+   https://github.com/restic/restic/issues/2923
+   https://github.com/restic/restic/pull/3513
+
+ * Enhancement #3729: Display full IDs in `check` warnings
+
+   When running commands to inspect or repair a damaged repository, it is often necessary to
+   supply the full IDs of objects stored in the repository.
+
+   The output of `check` now includes full IDs instead of their shortened variant.
+
+   https://github.com/restic/restic/pull/3729
+
+ * Enhancement #3773: Optimize memory usage for directories with many files
+
+   Backing up a directory with hundreds of thousands or more files caused restic to require large
+   amounts of memory. We've now optimized the `backup` command such that it requires up to 30% less
+   memory.
+
+   https://github.com/restic/restic/pull/3773
+
+ * Enhancement #3819: Validate include/exclude patterns before restoring
+
+   Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include` and
+   `--iinclude` weren't validated before running the restore. Invalid patterns would result in
+   error messages being printed repeatedly, and possibly unwanted files being restored.
+
+   Restic now validates all patterns before running the restore, and aborts with a fatal error if
+   an invalid pattern is detected.
+
+   https://github.com/restic/restic/pull/3819
+
+
 Changelog for restic 0.13.0 (2022-03-26)
 =======================================
 

CONTRIBUTING.md

@@ -48,9 +48,8 @@ environment was used and so on. Please tell us at least the following things:
 Remember, the easier it is for us to reproduce the bug, the earlier it will be
 corrected!
 
-In addition, you can compile restic with debug support by running
-`go run build.go -tags debug` and instructing it to create a debug
-log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
+In addition, you can instruct restic to create a debug log by setting the
+environment variable `DEBUG_LOG` to a file, e.g. like this:
 
     $ export DEBUG_LOG=/tmp/restic-debug.log
     $ restic backup ~/work
@@ -66,8 +65,8 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.14 or later for compiling. Clone the repo (without
-having `$GOPATH` set) and `cd` into the directory:
+Make sure you have the minimum required Go version installed. Clone the repo
+(without having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH
     $ git clone https://github.com/restic/restic

VERSION

@@ -1 +1 @@
-0.13.0
+0.14.0

changelog/0.14.0_2022-08-25/issue-1153

@@ -0,0 +1,9 @@
+Enhancement: Support pruning even when the disk is full
+
+When running out of disk space it was no longer possible to add or remove
+data from a repository. To help with recovering from such a deadlock, the
+prune command now supports an `--unsafe-recover-no-free-space` option to
+recover from these situations. Make sure to read the documentation first!
+
+https://github.com/restic/restic/issues/1153
+https://github.com/restic/restic/pull/3481

changelog/0.14.0_2022-08-25/issue-1842

@@ -0,0 +1,8 @@
+Change: Support debug log creation in release builds
+
+Creating a debug log was only possible in debug builds which required users to
+manually build restic. We changed the release builds to allow creating debug
+logs by simply setting the environment variable `DEBUG_LOG=logname.log`.
+
+https://github.com/restic/restic/issues/1842
+https://github.com/restic/restic/pull/3826

changelog/0.14.0_2022-08-25/issue-21

@@ -0,0 +1,28 @@
+Enhancement: Add compression support
+
+We've added compression support to the restic repository format. To create a
+repository using the new format run `init --repository-version 2`. Please note
+that the repository cannot be read by restic versions prior to 0.14.0.
+
+You can configure whether data is compressed with the option `--compression`. It
+can be set to `auto` (the default, which will compress very fast), `max` (which
+will trade backup speed and CPU usage for better compression), or `off` (which
+disables compression). Each setting is only applied for the current run of restic
+and does *not* apply to future runs. The option can also be set via the
+environment variable `RESTIC_COMPRESSION`.
+
+To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the
+documentation for more details. The migration checks the repository integrity
+and upgrades the repository format, but will not change any data. Afterwards,
+prune will rewrite the metadata to make use of compression.
+
+As an alternative you can use the `copy` command to migrate snapshots; First
+create a new repository using
+`init --repository-version 2 --copy-chunker-params --repo2 path/to/old/repo`,
+and then use the `copy` command to copy all snapshots to the new repository.
+
+https://github.com/restic/restic/issues/21
+https://github.com/restic/restic/issues/3779
+https://github.com/restic/restic/pull/3666
+https://github.com/restic/restic/pull/3704
+https://github.com/restic/restic/pull/3733

changelog/0.14.0_2022-08-25/issue-2162

@@ -0,0 +1,18 @@
+Enhancement: Adaptive IO concurrency based on backend connections
+
+Many commands used hard-coded limits for the number of concurrent operations.
+This prevented speed improvements by increasing the number of connections used
+by a backend.
+
+These limits have now been replaced by using the configured number of backend
+connections instead, which can be controlled using the
+`-o <backend-name>.connections=5` option. Commands will then automatically
+scale their parallelism accordingly.
+
+To limit the number of CPU cores used by restic, you can set the environment
+variable `GOMAXPROCS` accordingly. For example to use a single CPU core, use
+`GOMAXPROCS=1`.
+
+https://github.com/restic/restic/issues/2162
+https://github.com/restic/restic/issues/1467
+https://github.com/restic/restic/pull/3611

changelog/0.14.0_2022-08-25/issue-2248

@@ -0,0 +1,8 @@
+Bugfix: Support `self-update` on Windows
+
+Restic `self-update` would fail in situations where the operating system
+locks running binaries, including Windows. The new behavior works around
+this by renaming the running file and swapping the updated file in place.
+
+https://github.com/restic/restic/issues/2248
+https://github.com/restic/restic/pull/3675

changelog/0.14.0_2022-08-25/issue-2291

@@ -0,0 +1,12 @@
+Enhancement: Allow pack size customization
+
+Restic now uses a target pack size of 16 MiB by default. This can be customized
+using the `--pack-size size` option. Supported pack sizes range between 4 and
+128 MiB.
+
+It is possible to migrate an existing repository to _larger_ pack files using
+`prune --repack-small`. This will rewrite every pack file which is
+significantly smaller than the target size.
+
+https://github.com/restic/restic/issues/2291
+https://github.com/restic/restic/pull/3731

changelog/0.14.0_2022-08-25/issue-2295

@@ -0,0 +1,14 @@
+Enhancement: Allow use of SAS token to authenticate to Azure
+
+Previously restic only supported AccountKeys to authenticate to Azure
+storage accounts, which necessitates giving a significant amount of
+access.
+
+We added support for Azure SAS tokens which are a more fine-grained
+and time-limited manner of granting access. Set the `AZURE_ACCOUNT_NAME`
+and `AZURE_ACCOUNT_SAS` environment variables to use a SAS token for
+authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it will take
+precedence.
+
+https://github.com/restic/restic/issues/2295
+https://github.com/restic/restic/pull/3661

changelog/0.14.0_2022-08-25/issue-2696

@@ -0,0 +1,13 @@
+Enhancement: Improve backup speed with many small files
+
+We have restructured the backup pipeline to continue reading files while all
+upload connections are busy. This allows the backup to already prepare the next
+data file such that the upload can continue as soon as a connection becomes
+available. This can especially improve the backup performance for high latency
+backends.
+
+The upload concurrency is now controlled using the `-o <backend-name>.connections=5`
+option.
+
+https://github.com/restic/restic/issues/2696
+https://github.com/restic/restic/pull/3489

changelog/0.14.0_2022-08-25/issue-2907

@@ -0,0 +1,15 @@
+Enhancement: Make snapshot directory structure of `mount` command customizable
+
+We've added the possibility to customize the snapshot directory structure of
+the `mount` command using templates passed to the `--snapshot-template` option.
+The formatting of snapshots' timestamps is now controlled using `--time-template`
+and supports subdirectories to for example group snapshots by year. Please
+see `restic help mount` for further details.
+
+Characters in tag names which are not allowed in a filename are replaced by
+underscores `_`. For example a tag `foo/bar` will result in a directory name
+of `foo_bar`.
+
+https://github.com/restic/restic/issues/2907
+https://github.com/restic/restic/pull/2913
+https://github.com/restic/restic/pull/3691

changelog/0.14.0_2022-08-25/issue-3114

@@ -0,0 +1,12 @@
+Enhancement: Optimize handling of duplicate blobs in `prune`
+
+Restic `prune` always used to repack all data files containing duplicate
+blobs. This effectively removed all duplicates during prune. However, as a
+consequence all these data files were repacked even if the unused repository
+space threshold could be reached with less work.
+
+This is now changed and `prune` works nice and fast even when there are lots
+of duplicate blobs.
+
+https://github.com/restic/restic/issues/3114
+https://github.com/restic/restic/pull/3290

changelog/0.14.0_2022-08-25/issue-3295

@@ -0,0 +1,13 @@
+Change: Deprecate `check --check-unused` and add further checks
+
+Since restic 0.12.0, it is expected to still have unused blobs after running
+`prune`. This made the `--check-unused` option of the `check` command rather
+useless and tended to confuse users. This option has been deprecated and is
+now ignored.
+
+The `check` command now also warns if a repository is using either the legacy
+S3 layout or mixed pack files with both tree and data blobs. The latter is
+known to cause performance problems.
+
+https://github.com/restic/restic/issues/3295
+https://github.com/restic/restic/pull/3730

changelog/0.14.0_2022-08-25/issue-3428

@@ -0,0 +1,14 @@
+Bugfix: List snapshots in backend at most once to resolve snapshot IDs
+
+Many commands support specifying a list of snapshot IDs which are then used to
+determine the snapshots to be processed by the command. To resolve snapshot IDs
+or `latest`, and check that these exist, restic previously listed all snapshots
+stored in the repository. Depending on the backend this could be a slow and/or
+expensive operation.
+
+Restic now lists the snapshots only once and remembers the result in order to
+resolve all further snapshot IDs swiftly.
+
+https://github.com/restic/restic/issues/3428
+https://github.com/restic/restic/pull/3570
+https://github.com/restic/restic/pull/3395

changelog/0.14.0_2022-08-25/issue-3432

@@ -0,0 +1,14 @@
+Bugfix: Fix rare 'not found in repository' error for `copy` command
+
+In rare cases `copy` (and other commands) would report that `LoadTree(...)`
+returned an `id [...] not found in repository` error. This could be caused by
+a backup or copy command running concurrently. The error was only temporary;
+running the failed restic command a second time as a workaround did resolve the
+error.
+
+This issue has now been fixed by correcting the order in which restic reads data
+from the repository. It is now guaranteed that restic only loads snapshots for
+which all necessary data is already available.
+
+https://github.com/restic/restic/issues/3432
+https://github.com/restic/restic/pull/3570

changelog/0.14.0_2022-08-25/issue-3465

@@ -0,0 +1,10 @@
+Enhancement: Improve handling of temporary files on Windows
+
+In some cases restic failed to delete temporary files, causing the current
+command to fail. This has now been fixed by ensuring that Windows automatically
+deletes the file. In addition, temporary files are only written to disk when
+necessary, reducing disk writes.
+
+https://github.com/restic/restic/issues/3465
+https://github.com/restic/restic/issues/1551
+https://github.com/restic/restic/pull/3610

changelog/0.14.0_2022-08-25/issue-3685

@@ -0,0 +1,7 @@
+Bugfix: The `diff` command incorrectly listed some files as added
+
+There was a bug in the `diff` command, causing it to always show files in a
+removed directory as added. This has now been fixed.
+
+https://github.com/restic/restic/issues/3685
+https://github.com/restic/restic/pull/3686

changelog/0.14.0_2022-08-25/issue-3692

@@ -0,0 +1,13 @@
+Bugfix: Fix rclone (shimmed by Scoop) and sftp not working on Windows
+
+In #3602 a fix was introduced to address the problem of `rclone` prematurely
+exiting when Ctrl+C is pressed on Windows. The solution was to create the
+subprocess with its console detached from the restic console.
+
+However, this solution failed when using `rclone` installed by Scoop or using
+`sftp` with a passphrase-protected private key. We've now fixed this by using
+a different approach to prevent Ctrl-C from passing down too early.
+
+https://github.com/restic/restic/issues/3681
+https://github.com/restic/restic/issues/3692
+https://github.com/restic/restic/pull/3696

changelog/0.14.0_2022-08-25/issue-3709

@@ -0,0 +1,11 @@
+Enhancement: Validate exclude patterns before backing up
+
+Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or
+`--iexclude-file` previously weren't validated. As a consequence, invalid
+patterns resulted in files that were meant to be excluded being backed up.
+
+Restic now validates all patterns before running the backup and aborts with
+a fatal error if an invalid pattern is detected.
+
+https://github.com/restic/restic/issues/3709
+https://github.com/restic/restic/pull/3734

changelog/0.14.0_2022-08-25/issue-3720

@@ -0,0 +1,13 @@
+Bugfix: Directory sync errors for repositories accessed via SMB
+
+On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in
+restic failing to save the lock file, yielding the following errors:
+
+Save(<lock/071fe833f0>) returned error, retrying after 552.330144ms: sync /repo/locks: no such file or directory
+Save(<lock/bf789d7343>) returned error, retrying after 552.330144ms: sync /repo/locks: invalid argument
+
+This has now been fixed by ignoring the relevant error codes.
+
+https://github.com/restic/restic/issues/3720
+https://github.com/restic/restic/issues/3751
+https://github.com/restic/restic/pull/3752

changelog/0.14.0_2022-08-25/issue-3736

@@ -0,0 +1,8 @@
+Bugfix: The `stats` command miscalculated restore size for multiple snapshots
+
+Since restic 0.10.0 the restore size calculated by the `stats` command for
+multiple snapshots was too low. The hardlink detection was accidentally applied
+across multiple snapshots and thus ignored many files. This has now been fixed.
+
+https://github.com/restic/restic/issues/3736
+https://github.com/restic/restic/pull/3740

changelog/0.14.0_2022-08-25/issue-3837

@@ -0,0 +1,8 @@
+Enhancement: Improve SFTP repository initialization over slow links
+
+The `init` command, when used on an SFTP backend, now sends multiple `mkdir`
+commands to the backend concurrently. This reduces the waiting times when
+creating a repository over a very slow connection.
+
+https://github.com/restic/restic/issues/3837
+https://github.com/restic/restic/pull/3840

changelog/0.14.0_2022-08-25/issue-3861

@@ -0,0 +1,9 @@
+Bugfix: Yield error on invalid policy to `forget`
+
+The `forget` command previously silently ignored invalid/unsupported
+units in the duration options, such as e.g. `--keep-within-daily 2w`.
+
+Specifying an invalid/unsupported duration unit now results in an error.
+
+https://github.com/restic/restic/issues/3861
+https://github.com/restic/restic/pull/3862

changelog/0.14.0_2022-08-25/pull-3419

@@ -0,0 +1,21 @@
+Enhancement: Use config file permissions to control file group access
+
+Previously files in a local/SFTP repository would always end up with very
+restrictive access permissions, allowing access only to the owner. This
+prevented a number of valid use-cases involving groups and ACLs.
+
+We now use the permissions of the config file in the repository to decide
+whether group access should be given to newly created repository files or
+not. We arrange for repository files to be created group readable exactly
+when the repository config file is group readable.
+
+To opt-in to group readable repositories, a simple `chmod -R g+r` or
+equivalent on the config file can be used. For repositories that should
+be writable by group members a tad more setup is required, see the docs.
+
+Posix ACLs can also be used now that the group permissions being forced to
+zero no longer masks the effect of ACL entries.
+
+https://github.com/restic/restic/issues/2351
+https://github.com/restic/restic/pull/3419
+https://forum.restic.net/t/1391

changelog/0.14.0_2022-08-25/pull-3475

@@ -0,0 +1,9 @@
+Enhancement: Allow limiting IO concurrency for local and SFTP backend
+
+Restic did not support limiting the IO concurrency / number of connections for
+accessing repositories stored using the local or SFTP backends. The number of
+connections is now limited as for other backends, and can be configured via the
+the `-o local.connections=2` and `-o sftp.connections=5` options. This ensures
+that restic does not overwhelm the backend with concurrent IO operations.
+
+https://github.com/restic/restic/pull/3475

changelog/0.14.0_2022-08-25/pull-3484

@@ -0,0 +1,13 @@
+Enhancement: Stream data in `check` and `prune` commands
+
+The commands `check --read-data` and `prune` previously downloaded data files
+into temporary files which could end up being written to disk. This could cause
+a large amount of data being written to disk.
+
+The pack files are now instead streamed, which removes the need for temporary
+files. Please note that *uploads* during `backup` and `prune` still require
+temporary files.
+
+https://github.com/restic/restic/pull/3484
+https://github.com/restic/restic/issues/3710
+https://github.com/restic/restic/pull/3717

changelog/0.14.0_2022-08-25/pull-3513

@@ -0,0 +1,10 @@
+Enhancement: Improve speed of `copy` command
+
+The `copy` command could require a long time to copy snapshots for non-local
+backends. This has been improved to provide a throughput comparable to the
+`restore` command.
+
+Additionally, `copy` now displays a progress bar.
+
+https://github.com/restic/restic/issues/2923
+https://github.com/restic/restic/pull/3513

changelog/0.14.0_2022-08-25/pull-3680

@@ -0,0 +1,8 @@
+Change: Update dependencies and require Go 1.15 or newer
+
+We've updated most dependencies. Since some libraries require newer language
+features we're dropping support for Go 1.14, which means that restic now
+requires at least Go 1.15 to build.
+
+https://github.com/restic/restic/issues/3680
+https://github.com/restic/restic/issues/3883

changelog/0.14.0_2022-08-25/pull-3716

@@ -0,0 +1,7 @@
+Bugfix: Print "wrong password" to stderr instead of stdout
+
+If an invalid password was entered, the error message was printed on stdout and
+not on stderr as intended. This has now been fixed.
+
+https://github.com/restic/restic/pull/3716
+https://forum.restic.net/t/4965

changelog/0.14.0_2022-08-25/pull-3729

@@ -0,0 +1,8 @@
+Enhancement: Display full IDs in `check` warnings
+
+When running commands to inspect or repair a damaged repository, it is often
+necessary to supply the full IDs of objects stored in the repository.
+
+The output of `check` now includes full IDs instead of their shortened variant.
+
+https://github.com/restic/restic/pull/3729

changelog/0.14.0_2022-08-25/pull-3742

@@ -0,0 +1,19 @@
+Change: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
+
+The `init` and `copy` commands can read data from another repository.
+However, confusingly `--repo2` referred to the repository *from* which the
+`init` command copies parameters, but for the `copy` command `--repo2`
+referred to the copy *destination*.
+
+We've introduced a new option, `--from-repo`, which always refers to the
+source repository for both commands. The old parameter names have been
+deprecated but still work. To create a new repository and copy all snapshots
+to it, the commands are now as follows:
+
+```
+restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params
+restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo
+```
+
+https://github.com/restic/restic/pull/3742
+https://forum.restic.net/t/5017

changelog/0.14.0_2022-08-25/pull-3772

@@ -0,0 +1,14 @@
+Bugfix: Correctly rebuild index for legacy repositories
+
+After running `rebuild-index` on a legacy repository containing mixed pack
+files (that is, pack files which store both metadata and file data), `check`
+printed warnings like `pack 12345678 contained in several indexes: ...`.
+This warning was not critical, but has now nonetheless been fixed by properly
+handling mixed pack files while rebuilding the index.
+
+Running `prune` for such legacy repositories will also fix the warning by
+reorganizing the pack files which caused it.
+
+https://github.com/restic/restic/pull/3772
+https://github.com/restic/restic/pull/3884
+https://forum.restic.net/t/5044/13

changelog/0.14.0_2022-08-25/pull-3773

@@ -0,0 +1,7 @@
+Enhancement: Optimize memory usage for directories with many files
+
+Backing up a directory with hundreds of thousands or more files caused restic
+to require large amounts of memory. We've now optimized the `backup` command
+such that it requires up to 30% less memory.
+
+https://github.com/restic/restic/pull/3773

changelog/0.14.0_2022-08-25/pull-3776

@@ -0,0 +1,11 @@
+Bugfix: Limit number of key files tested while opening a repository
+
+Previously, restic tested the password against every key in the repository
+when opening a repository. The more keys there were in the repository, the
+slower this operation became.
+
+Restic now tests the password against up to 20 key files in the repository.
+Alternatively, you can use the `--key-hint=<key ID>` option to specify a
+specific key file to use instead.
+
+https://github.com/restic/restic/pull/3776

changelog/0.14.0_2022-08-25/pull-3819

@@ -0,0 +1,11 @@
+Enhancement: Validate include/exclude patterns before restoring
+
+Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include`
+and `--iinclude` weren't validated before running the restore. Invalid
+patterns would result in error messages being printed repeatedly, and
+possibly unwanted files being restored.
+
+Restic now validates all patterns before running the restore, and aborts
+with a fatal error if an invalid pattern is detected.
+
+https://github.com/restic/restic/pull/3819

changelog/TEMPLATE

@@ -1,5 +1,5 @@
 # The first line must start with Bugfix:, Enhancement: or Change:,
-# including the colon. Use present use. Remove lines starting with '#'
+# including the colon. Use present tense. Remove lines starting with '#'
 # from this template.
 Enhancement: Allow custom bar in the foo command
 

cmd/restic/cmd_backup.go

@@ -12,14 +12,16 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
-	tomb "gopkg.in/tomb.v2"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -29,7 +31,7 @@ import (
 )
 
 var cmdBackup = &cobra.Command{
-	Use:   "backup [flags] FILE/DIR [FILE/DIR] ...",
+	Use:   "backup [flags] [FILE/DIR] ...",
 	Short: "Create a new backup of files and/or directories",
 	Long: `
 The "backup" command creates a new snapshot and saves the files and directories
@@ -54,16 +56,22 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		var t tomb.Tomb
-		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
-		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
+		var wg sync.WaitGroup
+		cancelCtx, cancel := context.WithCancel(globalOptions.ctx)
+		defer func() {
+			// shutdown termstatus
+			cancel()
+			wg.Wait()
+		}()
 
-		err := runBackup(backupOptions, globalOptions, term, args)
-		t.Kill(nil)
-		if werr := t.Wait(); werr != nil {
-			panic(fmt.Sprintf("term.Run() returned err: %v", err))
-		}
-		return err
+		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			term.Run(cancelCtx)
+		}()
+
+		return runBackup(backupOptions, globalOptions, term, args)
 	},
 }
 
@@ -103,7 +111,7 @@ func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
@@ -143,7 +151,7 @@ func init() {
 func filterExisting(items []string) (result []string, err error) {
 	for _, item := range items {
 		_, err := fs.Lstat(item)
-		if err != nil && os.IsNotExist(errors.Cause(err)) {
+		if errors.Is(err, os.ErrNotExist) {
 			Warnf("%v does not exist, skipping\n", item)
 			continue
 		}
@@ -298,6 +306,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 		if err != nil {
 			return nil, err
 		}
+
+		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
+			return nil, errors.Fatalf("--exclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		opts.Excludes = append(opts.Excludes, excludes...)
 	}
 
@@ -306,14 +319,27 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 		if err != nil {
 			return nil, err
 		}
+
+		if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
+			return nil, errors.Fatalf("--iexclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
 	}
 
 	if len(opts.InsensitiveExcludes) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExcludes); !valid {
+			return nil, errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
 	}
 
 	if len(opts.Excludes) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.Excludes); !valid {
+			return nil, errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+
 		fs = append(fs, rejectByPattern(opts.Excludes))
 	}
 
@@ -475,7 +501,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
-		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
+		id, err := restic.FindSnapshot(ctx, repo.Backend(), opts.Parent)
 		if err != nil {
 			return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
 		}
@@ -485,7 +511,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
+		id, err := restic.FindLatestSnapshot(ctx, repo.Backend(), repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -515,8 +541,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 	}
 
-	var t tomb.Tomb
-
 	if gopts.verbosity >= 2 && !gopts.JSON {
 		Verbosef("open repository\n")
 	}
@@ -548,7 +572,10 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 
 	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 
-	t.Go(func() error { return progressReporter.Run(t.Context(gopts.ctx)) })
+	wg, wgCtx := errgroup.WithContext(gopts.ctx)
+	cancelCtx, cancel := context.WithCancel(wgCtx)
+	defer cancel()
+	wg.Go(func() error { return progressReporter.Run(cancelCtx) })
 
 	if !gopts.JSON {
 		progressPrinter.V("lock repository")
@@ -571,14 +598,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	if !gopts.JSON {
-		progressPrinter.V("load index files")
-	}
-	err = repo.LoadIndex(gopts.ctx)
-	if err != nil {
-		return err
-	}
-
 	var parentSnapshotID *restic.ID
 	if !opts.Stdin {
 		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
@@ -595,6 +614,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 	}
 
+	if !gopts.JSON {
+		progressPrinter.V("load index files")
+	}
+	err = repo.LoadIndex(gopts.ctx)
+	if err != nil {
+		return err
+	}
+
 	selectByNameFilter := func(item string) bool {
 		for _, reject := range rejectByNameFuncs {
 			if reject(item) {
@@ -620,7 +647,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 
 		errorHandler := func(item string, err error) error {
-			return progressReporter.Error(item, nil, err)
+			return progressReporter.Error(item, err)
 		}
 
 		messageHandler := func(msg string, args ...interface{}) {
@@ -656,16 +683,16 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	if !gopts.JSON {
 		progressPrinter.V("start scan on %v", targets)
 	}
-	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
+	wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 
 	arch := archiver.New(repo, targetFS, archiver.Options{})
 	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
 	success := true
-	arch.Error = func(item string, fi os.FileInfo, err error) error {
+	arch.Error = func(item string, err error) error {
 		success = false
-		return progressReporter.Error(item, fi, err)
+		return progressReporter.Error(item, err)
 	}
 	arch.CompleteItem = progressReporter.CompleteItem
 	arch.StartFile = progressReporter.StartFile
@@ -698,10 +725,10 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
 
 	// cleanly shutdown all running goroutines
-	t.Kill(nil)
+	cancel()
 
 	// let's see if one returned an error
-	werr := t.Wait()
+	werr := wg.Wait()
 
 	// return original error
 	if err != nil {

cmd/restic/cmd_cat.go

@@ -62,7 +62,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			}
 
 			// find snapshot id with prefix
-			id, err = restic.FindSnapshot(gopts.ctx, repo, args[1])
+			id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1])
 			if err != nil {
 				return errors.Fatalf("could not find snapshot: %v\n", err)
 			}
@@ -79,7 +79,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadAndDecrypt(gopts.ctx, nil, restic.IndexFile, id)
+		buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil)
 		if err != nil {
 			return err
 		}
@@ -87,13 +87,12 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn := &restic.Snapshot{}
-		err = repo.LoadJSONUnpacked(gopts.ctx, restic.SnapshotFile, id, sn)
+		sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
 		if err != nil {
 			return err
 		}
 
-		buf, err := json.MarshalIndent(&sn, "", "  ")
+		buf, err := json.MarshalIndent(sn, "", "  ")
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_check.go

@@ -57,7 +57,13 @@ func init() {
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
 	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
-	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
+	var ignored bool
+	f.BoolVar(&ignored, "check-unused", false, "find unused blobs")
+	err := f.MarkDeprecated("check-unused", "`--check-unused` is deprecated and will be ignored")
+	if err != nil {
+		// MarkDeprecated only returns an error when the flag is not found
+		panic(err)
+	}
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
 
@@ -142,10 +148,10 @@ func parsePercentage(s string) (float64, error) {
 
 // prepareCheckCache configures a special cache directory for check.
 //
-//  * if --with-cache is specified, the default cache is used
-//  * if the user explicitly requested --no-cache, we don't use any cache
-//  * if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
-//  * by default, we use a cache in a temporary directory that is deleted after the check
+//   - if --with-cache is specified, the default cache is used
+//   - if the user explicitly requested --no-cache, we don't use any cache
+//   - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
+//   - by default, we use a cache in a temporary directory that is deleted after the check
 func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
 	cleanup = func() {}
 	if opts.WithCache {
@@ -211,21 +217,37 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	chkr := checker.New(repo, opts.CheckUnused)
+	err = chkr.LoadSnapshots(gopts.ctx)
+	if err != nil {
+		return err
+	}
 
 	Verbosef("load indexes\n")
 	hints, errs := chkr.LoadIndex(gopts.ctx)
 
-	dupFound := false
+	errorsFound := false
+	suggestIndexRebuild := false
+	mixedFound := false
 	for _, hint := range hints {
-		Printf("%v\n", hint)
-		if _, ok := hint.(checker.ErrDuplicatePacks); ok {
-			dupFound = true
+		switch hint.(type) {
+		case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
+			Printf("%v\n", hint)
+			suggestIndexRebuild = true
+		case *checker.ErrMixedPack:
+			Printf("%v\n", hint)
+			mixedFound = true
+		default:
+			Warnf("error: %v\n", hint)
+			errorsFound = true
 		}
 	}
 
-	if dupFound {
+	if suggestIndexRebuild {
 		Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
 	}
+	if mixedFound {
+		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
+	}
 
 	if len(errs) > 0 {
 		for _, err := range errs {
@@ -234,7 +256,6 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("LoadIndex returned errors")
 	}
 
-	errorsFound := false
 	orphanedPacks := 0
 	errChan := make(chan error)
 
@@ -245,14 +266,16 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		if checker.IsOrphanedPack(err) {
 			orphanedPacks++
 			Verbosef("%v\n", err)
-			continue
+		} else if _, ok := err.(*checker.ErrLegacyLayout); ok {
+			Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
+		} else {
+			errorsFound = true
+			Warnf("%v\n", err)
 		}
-		errorsFound = true
-		Warnf("%v\n", err)
 	}
 
 	if orphanedPacks > 0 {
-		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nYou can run `restic prune` to correct this.\n", orphanedPacks)
+		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
 	}
 
 	Verbosef("check snapshots, trees and blobs\n")
@@ -269,7 +292,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	for err := range errChan {
 		errorsFound = true
-		if e, ok := err.(checker.TreeError); ok {
+		if e, ok := err.(*checker.TreeError); ok {
 			Warnf("error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
 				Warnf("  %v\n", treeErr)

cmd/restic/cmd_copy.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
 
@@ -48,17 +50,21 @@ func init() {
 	cmdRoot.AddCommand(cmdCopy)
 
 	f := cmdCopy.Flags()
-	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots to")
+	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
 	f.StringArrayVarP(&copyOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
 	f.Var(&copyOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
 	f.StringArrayVar(&copyOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 }
 
 func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
-	dstGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
+	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
 	if err != nil {
 		return err
 	}
+	if isFromRepo {
+		// swap global options, if the secondary repo was set via from-repo
+		gopts, secondaryGopts = secondaryGopts, gopts
+	}
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
@@ -68,7 +74,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	dstRepo, err := OpenRepository(dstGopts)
+	dstRepo, err := OpenRepository(secondaryGopts)
 	if err != nil {
 		return err
 	}
@@ -87,6 +93,16 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
+	srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
+	dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	debug.Log("Loading source index")
 	if err := srcRepo.LoadIndex(ctx); err != nil {
 		return err
@@ -98,7 +114,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
-	for sn := range FindFilteredSnapshots(ctx, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
+	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) {
 		if sn.Original != nil && !sn.Original.IsNull() {
 			dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
 		}
@@ -109,7 +125,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 	// remember already processed trees across all snapshots
 	visitedTrees := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
 
 		// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
@@ -131,24 +147,18 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 			}
 		}
 		Verbosef("  copy started, this may take a while...\n")
-
-		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree); err != nil {
+		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
 			return err
 		}
 		debug.Log("tree copied")
 
-		if err = dstRepo.Flush(ctx); err != nil {
-			return err
-		}
-		debug.Log("flushed packs and saved index")
-
 		// save snapshot
 		sn.Parent = nil // Parent does not have relevance in the new repo.
 		// Use Original as a persistent snapshot ID
 		if sn.Original == nil {
 			sn.Original = sn.ID()
 		}
-		newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+		newID, err := restic.SaveSnapshot(ctx, dstRepo, sn)
 		if err != nil {
 			return err
 		}
@@ -176,82 +186,61 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 
-const numCopyWorkers = 8
-
 func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
-	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
+	visitedTrees restic.IDSet, rootTreeID restic.ID, quiet bool) error {
 
-	idChan := make(chan restic.ID)
-	wg, ctx := errgroup.WithContext(ctx)
+	wg, wgCtx := errgroup.WithContext(ctx)
 
-	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
+	treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
 		visited := visitedTrees.Has(treeID)
 		visitedTrees.Insert(treeID)
 		return visited
 	}, nil)
 
+	copyBlobs := restic.NewBlobSet()
+	packList := restic.NewIDSet()
+
+	enqueue := func(h restic.BlobHandle) {
+		pb := srcRepo.Index().Lookup(h)
+		copyBlobs.Insert(h)
+		for _, p := range pb {
+			packList.Insert(p.PackID)
+		}
+	}
+
 	wg.Go(func() error {
-		defer close(idChan)
-		// reused buffer
-		var buf []byte
 		for tree := range treeStream {
 			if tree.Error != nil {
 				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
 			}
 
 			// Do we already have this tree blob?
-			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
+			treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
+			if !dstRepo.Index().Has(treeHandle) {
 				// copy raw tree bytes to avoid problems if the serialization changes
-				var err error
-				buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
-				if err != nil {
-					return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
-				}
-
-				_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
-				if err != nil {
-					return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
-				}
+				enqueue(treeHandle)
 			}
 
 			for _, entry := range tree.Nodes {
 				// Recursion into directories is handled by StreamTrees
 				// Copy the blobs for this file.
 				for _, blobID := range entry.Content {
-					select {
-					case idChan <- blobID:
-					case <-ctx.Done():
-						return ctx.Err()
+					h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
+					if !dstRepo.Index().Has(h) {
+						enqueue(h)
 					}
 				}
 			}
 		}
 		return nil
 	})
-
-	for i := 0; i < numCopyWorkers; i++ {
-		wg.Go(func() error {
-			// reused buffer
-			var buf []byte
-			for blobID := range idChan {
-				// Do we already have this data blob?
-				if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
-					continue
-				}
-				debug.Log("Copying blob %s\n", blobID.Str())
-				var err error
-				buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
-				if err != nil {
-					return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
-				}
-
-				_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
-				if err != nil {
-					return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
-				}
-			}
-			return nil
-		})
+	err := wg.Wait()
+	if err != nil {
+		return err
 	}
-	return wg.Wait()
+
+	bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
+	_, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar)
+	bar.Done()
+	return err
 }

cmd/restic/cmd_debug.go

@@ -1,3 +1,4 @@
+//go:build debug
 // +build debug
 
 package main
@@ -14,6 +15,7 @@ import (
 	"sort"
 	"time"
 
+	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 
@@ -51,12 +53,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 var tryRepair bool
 var repairByte bool
 var extractPack bool
+var reuploadBlobs bool
 
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
 	cmdDebug.AddCommand(cmdDebugExamine)
 	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
+	cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
 	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
 	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
@@ -72,7 +76,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 }
 
 func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
-	return restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
+	return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
 		if err != nil {
 			return err
 		}
@@ -103,7 +107,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 	return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 
-		blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
+		blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
 			return nil
@@ -308,6 +312,10 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
 }
 
 func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
+	dec, err := zstd.NewReader(nil)
+	if err != nil {
+		panic(err)
+	}
 	be := repo.Backend()
 	h := restic.Handle{
 		Name: pack.String(),
@@ -332,48 +340,65 @@ func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list
 
 		nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
 		plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+		outputPrefix := ""
+		filePrefix := ""
 		if err != nil {
 			Warnf("error decrypting blob: %v\n", err)
-			var plain []byte
 			if tryRepair || repairByte {
-				plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
+				plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
 			}
-			var prefix string
-			if plain != nil {
-				id := restic.Hash(plain)
-				if !id.Equal(blob.ID) {
-					Printf("         repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
-					prefix = "repaired-wrong-hash-"
-				} else {
-					Printf("         successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
-					prefix = "repaired-"
-				}
+			if plaintext != nil {
+				outputPrefix = "repaired "
+				filePrefix = "repaired-"
 			} else {
-				plain = decryptUnsigned(ctx, key, buf)
-				prefix = "damaged-"
+				plaintext = decryptUnsigned(ctx, key, buf)
+				err = storePlainBlob(blob.ID, "damaged-", plaintext)
+				if err != nil {
+					return err
+				}
+				continue
 			}
-			err = storePlainBlob(blob.ID, prefix, plain)
+		}
+
+		if blob.IsCompressed() {
+			decompressed, err := dec.DecodeAll(plaintext, nil)
 			if err != nil {
-				return err
+				Printf("         failed to decompress blob %v\n", blob.ID)
+			}
+			if decompressed != nil {
+				plaintext = decompressed
 			}
-			continue
 		}
 
 		id := restic.Hash(plaintext)
 		var prefix string
 		if !id.Equal(blob.ID) {
-			Printf("         successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
+			Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
 			prefix = "wrong-hash-"
 		} else {
-			Printf("         successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
+			Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
 			prefix = "correct-"
 		}
 		if extractPack {
-			err = storePlainBlob(id, prefix, plaintext)
+			err = storePlainBlob(id, filePrefix+prefix, plaintext)
 			if err != nil {
 				return err
 			}
 		}
+		if reuploadBlobs {
+			_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
+			if err != nil {
+				return err
+			}
+			Printf("         uploaded %v %v\n", blob.Type, id)
+		}
+	}
+
+	if reuploadBlobs {
+		err := repo.Flush(ctx)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -402,12 +427,23 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 }
 
 func runDebugExamine(gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
 	ids := make([]restic.ID, 0)
 	for _, name := range args {
 		id, err := restic.ParseID(name)
 		if err != nil {
-			Warnf("error: %v\n", err)
-			continue
+			name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name)
+			if err == nil {
+				id, err = restic.ParseID(name)
+			}
+			if err != nil {
+				Warnf("error: %v\n", err)
+				continue
+			}
 		}
 		ids = append(ids, id)
 	}
@@ -416,11 +452,6 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no pack files to examine")
 	}
 
-	repo, err := OpenRepository(gopts)
-	if err != nil {
-		return err
-	}
-
 	if !gopts.NoLock {
 		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
@@ -475,27 +506,15 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 
 	blobsLoaded := false
 	// examine all data the indexes have for the pack file
-	for _, idx := range repo.Index().(*repository.MasterIndex).All() {
-		idxIDs, err := idx.IDs()
-		if err != nil {
-			idxIDs = restic.IDs{}
-		}
-
-		blobs := idx.ListPack(id)
+	for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) {
+		blobs := b.Blobs
 		if len(blobs) == 0 {
 			continue
 		}
 
-		Printf("    index %v:\n", idxIDs)
+		checkPackSize(blobs, fi.Size)
 
-		// convert list of blobs to []restic.Blob
-		var list []restic.Blob
-		for _, b := range blobs {
-			list = append(list, b.Blob)
-		}
-		checkPackSize(list, fi.Size)
-
-		err = loadBlobs(ctx, repo, id, list)
+		err = loadBlobs(ctx, repo, id, blobs)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		} else {
@@ -506,7 +525,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 	Printf("  ========================================\n")
 	Printf("  inspect the pack itself\n")
 
-	blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
+	blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size)
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}
@@ -531,14 +550,10 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) {
 		if offset != uint64(pb.Offset) {
 			Printf("      hole in file, want offset %v, got %v\n", offset, pb.Offset)
 		}
-		offset += uint64(pb.Length)
+		offset = uint64(pb.Offset + pb.Length)
 		size += uint64(pb.Length)
 	}
-
-	// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
-	size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
-	// length in uint32 little endian
-	size += 4
+	size += uint64(pack.CalculateHeaderSize(blobs))
 
 	if uint64(fileSize) != size {
 		Printf("      file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)

cmd/restic/cmd_diff.go

@@ -7,9 +7,9 @@ import (
 	"reflect"
 	"sort"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -53,8 +53,8 @@ func init() {
 	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
-func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
-	id, err := restic.FindSnapshot(ctx, repo, desc)
+func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
+	id, err := restic.FindSnapshot(ctx, be, desc)
 	if err != nil {
 		return nil, errors.Fatal(err.Error())
 	}
@@ -160,7 +160,7 @@ func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat)
 
 func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
 	debug.Log("print %v tree %v", mode, id)
-	tree, err := c.repo.LoadTree(ctx, id)
+	tree, err := restic.LoadTree(ctx, c.repo, id)
 	if err != nil {
 		return err
 	}
@@ -170,7 +170,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		if node.Type == "dir" {
 			name += "/"
 		}
-		c.printChange(NewChange(name, "+"))
+		c.printChange(NewChange(name, mode))
 		stats.Add(node)
 		addBlobs(blobs, node)
 
@@ -187,7 +187,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 
 func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
 	debug.Log("print tree %v", id)
-	tree, err := c.repo.LoadTree(ctx, id)
+	tree, err := restic.LoadTree(ctx, c.repo, id)
 	if err != nil {
 		return err
 	}
@@ -231,12 +231,12 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
 
 func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
-	tree1, err := c.repo.LoadTree(ctx, id1)
+	tree1, err := restic.LoadTree(ctx, c.repo, id1)
 	if err != nil {
 		return err
 	}
 
-	tree2, err := c.repo.LoadTree(ctx, id2)
+	tree2, err := restic.LoadTree(ctx, c.repo, id2)
 	if err != nil {
 		return err
 	}
@@ -334,10 +334,6 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if err = repo.LoadIndex(ctx); err != nil {
-		return err
-	}
-
 	if !gopts.NoLock {
 		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
@@ -346,12 +342,17 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	sn1, err := loadSnapshot(ctx, repo, args[0])
+	// cache snapshots listing
+	be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+	sn1, err := loadSnapshot(ctx, be, repo, args[0])
 	if err != nil {
 		return err
 	}
 
-	sn2, err := loadSnapshot(ctx, repo, args[1])
+	sn2, err := loadSnapshot(ctx, be, repo, args[1])
 	if err != nil {
 		return err
 	}
@@ -360,6 +361,10 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
 	}
 
+	if err = repo.LoadIndex(ctx); err != nil {
+		return err
+	}
+
 	if sn1.Tree == nil {
 		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
 	}

cmd/restic/cmd_dump.go

@@ -87,7 +87,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 			case l == 1 && dump.IsFile(node):
 				return d.WriteNode(ctx, node)
 			case l > 1 && dump.IsDir(node):
-				subtree, err := repo.LoadTree(ctx, *node.Subtree)
+				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
@@ -96,7 +96,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err := checkStdoutArchive(); err != nil {
 					return err
 				}
-				subtree, err := repo.LoadTree(ctx, *node.Subtree)
+				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
 				if err != nil {
 					return err
 				}
@@ -144,20 +144,15 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex(ctx)
-	if err != nil {
-		return err
-	}
-
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
+		id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
-		id, err = restic.FindSnapshot(ctx, repo, snapshotIDString)
+		id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString)
 		if err != nil {
 			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
 		}
@@ -168,7 +163,12 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	tree, err := repo.LoadTree(ctx, *sn.Tree)
+	err = repo.LoadIndex(ctx)
+	if err != nil {
+		return err
+	}
+
+	tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
 	if err != nil {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}

cmd/restic/cmd_find.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
@@ -584,6 +585,11 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	if err = repo.LoadIndex(gopts.ctx); err != nil {
 		return err
 	}
@@ -618,7 +624,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
 		if f.blobIDs != nil || f.treeIDs != nil {
 			if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
 				return err

cmd/restic/cmd_forget.go

@@ -14,11 +14,16 @@ var cmdForget = &cobra.Command{
 	Use:   "forget [flags] [snapshot ID] [...]",
 	Short: "Remove snapshots from the repository",
 	Long: `
-The "forget" command removes snapshots according to a policy. Please note that
-this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove the unreferenced data
-after "forget" was run successfully, see the "prune" command. Please also read
-the documentation for "forget" to learn about important security considerations.
+The "forget" command removes snapshots according to a policy. All snapshots are
+first divided into groups according to "--group-by", and after that the policy
+specified by the "--keep-*" options is applied to each group individually.
+
+Please note that this command really only deletes the snapshot object in the
+repository, which is a reference to data stored there. In order to remove the
+unreferenced data after "forget" was run successfully, see the "prune" command.
+
+Please also read the documentation for "forget" to learn about some important
+security considerations.
 
 EXIT STATUS
 ===========
@@ -91,7 +96,7 @@ func init() {
 	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
 
-	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "string for grouping snapshots by host,paths,tags")
+	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
@@ -128,7 +133,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		snapshots = append(snapshots, sn)
 	}
 

cmd/restic/cmd_init.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"strconv"
+
 	"github.com/restic/chunker"
 	"github.com/restic/restic/internal/backend/location"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 )
@@ -30,6 +33,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type InitOptions struct {
 	secondaryRepoOptions
 	CopyChunkerParameters bool
+	RepositoryVersion     string
 }
 
 var initOptions InitOptions
@@ -40,9 +44,26 @@ func init() {
 	f := cmdInit.Flags()
 	initSecondaryRepoOptions(f, &initOptions.secondaryRepoOptions, "secondary", "to copy chunker parameters from")
 	f.BoolVar(&initOptions.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)")
+	f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
 }
 
 func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
+	var version uint
+	if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
+		version = restic.MaxRepoVersion
+	} else if opts.RepositoryVersion == "stable" {
+		version = restic.StableRepoVersion
+	} else {
+		v, err := strconv.ParseUint(opts.RepositoryVersion, 10, 32)
+		if err != nil {
+			return errors.Fatal("invalid repository version")
+		}
+		version = uint(v)
+	}
+	if version < restic.MinRepoVersion || version > restic.MaxRepoVersion {
+		return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
+	}
+
 	chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
 	if err != nil {
 		return err
@@ -65,9 +86,15 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 
-	s := repository.New(be)
+	s, err := repository.New(be, repository.Options{
+		Compression: gopts.Compression,
+		PackSize:    gopts.PackSize * 1024 * 1024,
+	})
+	if err != nil {
+		return err
+	}
 
-	err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)
+	err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial)
 	if err != nil {
 		return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
@@ -83,7 +110,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 
 func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
 	if opts.CopyChunkerParameters {
-		otherGopts, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
+		otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
 		if err != nil {
 			return nil, err
 		}
@@ -97,7 +124,7 @@ func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker
 		return &pol, nil
 	}
 
-	if opts.Repo != "" {
+	if opts.Repo != "" || opts.RepositoryFile != "" || opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" {
 		return nil, errors.Fatal("Secondary repository must only be specified when copying the chunker parameters")
 	}
 	return nil, nil

cmd/restic/cmd_ls.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -169,6 +170,11 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
+	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	if err = repo.LoadIndex(gopts.ctx); err != nil {
 		return err
 	}
@@ -211,7 +217,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
 		printSnapshot(sn)
 
 		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {

cmd/restic/cmd_migrate.go

@@ -8,11 +8,12 @@ import (
 )
 
 var cmdMigrate = &cobra.Command{
-	Use:   "migrate [flags] [name]",
+	Use:   "migrate [flags] [migration name] [...]",
 	Short: "Apply migrations",
 	Long: `
-The "migrate" command applies migrations to a repository. When no migration
-name is explicitly given, a list of migrations that can be applied is printed.
+The "migrate" command checks which migrations can be applied for a repository
+and prints a list with available migration names. If one or more migration
+names are specified, these migrations are applied.
 
 EXIT STATUS
 ===========
@@ -41,6 +42,8 @@ func init() {
 func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
 	ctx := gopts.ctx
 	Printf("available migrations:\n")
+	found := false
+
 	for _, m := range migrations.All {
 		ok, err := m.Check(ctx, repo)
 		if err != nil {
@@ -48,10 +51,15 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 		}
 
 		if ok {
-			Printf("  %v: %v\n", m.Name(), m.Desc())
+			Printf("  %v\t%v\n", m.Name(), m.Desc())
+			found = true
 		}
 	}
 
+	if !found {
+		Printf("no migrations found")
+	}
+
 	return nil
 }
 
@@ -76,6 +84,19 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
 					Warnf("check for migration %v failed, continuing anyway\n", m.Name())
 				}
 
+				if m.RepoCheck() {
+					Printf("checking repository integrity...\n")
+
+					checkOptions := CheckOptions{}
+					checkGopts := gopts
+					// the repository is already locked
+					checkGopts.NoLock = true
+					err = runCheck(checkOptions, checkGopts, []string{})
+					if err != nil {
+						return err
+					}
+				}
+
 				Printf("applying migration %v...\n", m.Name())
 				if err = m.Apply(ctx, repo); err != nil {
 					Warnf("migration %v failed: %v\n", m.Name(), err)

cmd/restic/cmd_mount.go

@@ -1,3 +1,4 @@
+//go:build darwin || freebsd || linux
 // +build darwin freebsd linux
 
 package main
@@ -30,10 +31,13 @@ read-only mount.
 Snapshot Directories
 ====================
 
-If you need a different template for all directories that contain snapshots,
-you can pass a template via --snapshot-template. Example without colons:
+If you need a different template for directories that contain snapshots,
+you can pass a time template via --time-template and path templates via
+--path-template.
 
-    --snapshot-template "2006-01-02_15-04-05"
+Example time template without colons:
+
+    --time-template "2006-01-02_15-04-05"
 
 You need to specify a sample format for exactly the following timestamp:
 
@@ -42,6 +46,20 @@ You need to specify a sample format for exactly the following timestamp:
 For details please see the documentation for time.Format() at:
   https://godoc.org/time#Time.Format
 
+For path templates, you can use the following patterns which will be replaced:
+    %i by short snapshot ID
+    %I by long snapshot ID
+    %u by username
+    %h by hostname
+    %t by tags
+    %T by timestamp as specified by --time-template
+
+The default path templates are:
+    "ids/%i"
+    "snapshots/%T"
+    "hosts/%h/%T"
+    "tags/%t/%T"
+
 EXIT STATUS
 ===========
 
@@ -61,7 +79,8 @@ type MountOptions struct {
 	Hosts                []string
 	Tags                 restic.TagLists
 	Paths                []string
-	SnapshotTemplate     string
+	TimeTemplate         string
+	PathTemplates        []string
 }
 
 var mountOptions MountOptions
@@ -78,16 +97,21 @@ func init() {
 	mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
 	mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 
-	mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
+	mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
+	mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
+	mountFlags.StringVar(&mountOptions.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times")
+	_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
 }
 
 func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
-	if opts.SnapshotTemplate == "" {
-		return errors.Fatal("snapshot template string cannot be empty")
+	if opts.TimeTemplate == "" {
+		return errors.Fatal("time template string cannot be empty")
 	}
-	if strings.ContainsAny(opts.SnapshotTemplate, `\/`) {
-		return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character")
+
+	if strings.HasPrefix(opts.TimeTemplate, "/") || strings.HasSuffix(opts.TimeTemplate, "/") {
+		return errors.Fatal("time template string cannot start or end with '/'")
 	}
+
 	if len(args) == 0 {
 		return errors.Fatal("wrong number of parameters")
 	}
@@ -115,7 +139,7 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 
 	mountpoint := args[0]
 
-	if _, err := resticfs.Stat(mountpoint); os.IsNotExist(errors.Cause(err)) {
+	if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
 		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
 		return err
 	}
@@ -153,11 +177,12 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	cfg := fuse.Config{
-		OwnerIsRoot:      opts.OwnerRoot,
-		Hosts:            opts.Hosts,
-		Tags:             opts.Tags,
-		Paths:            opts.Paths,
-		SnapshotTemplate: opts.SnapshotTemplate,
+		OwnerIsRoot:   opts.OwnerRoot,
+		Hosts:         opts.Hosts,
+		Tags:          opts.Tags,
+		Paths:         opts.Paths,
+		TimeTemplate:  opts.TimeTemplate,
+		PathTemplates: opts.PathTemplates,
 	}
 	root := fuse.NewRoot(repo, cfg)
 

cmd/restic/cmd_prune.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"math"
 	"sort"
 	"strconv"
@@ -8,6 +9,7 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 
@@ -38,7 +40,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // PruneOptions collects all options for the cleanup command.
 type PruneOptions struct {
-	DryRun bool
+	DryRun                bool
+	UnsafeNoSpaceRecovery string
+
+	unsafeRecovery bool
 
 	MaxUnused      string
 	maxUnusedBytes func(used uint64) (unused uint64) // calculates the number of unused bytes after repacking, according to MaxUnused
@@ -47,6 +52,8 @@ type PruneOptions struct {
 	MaxRepackBytes uint64
 
 	RepackCachableOnly bool
+	RepackSmall        bool
+	RepackUncompressed bool
 }
 
 var pruneOptions PruneOptions
@@ -55,6 +62,7 @@ func init() {
 	cmdRoot.AddCommand(cmdPrune)
 	f := cmdPrune.Flags()
 	f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
+	f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
 	addPruneOptions(cmdPrune)
 }
 
@@ -63,6 +71,8 @@ func addPruneOptions(c *cobra.Command) {
 	f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
 	f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
 	f.BoolVar(&pruneOptions.RepackCachableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
+	f.BoolVar(&pruneOptions.RepackSmall, "repack-small", false, "repack pack files below 80% of target pack size")
+	f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data")
 }
 
 func verifyPruneOptions(opts *PruneOptions) error {
@@ -74,6 +84,10 @@ func verifyPruneOptions(opts *PruneOptions) error {
 		}
 		opts.MaxRepackBytes = uint64(size)
 	}
+	if opts.UnsafeNoSpaceRecovery != "" {
+		// prevent repacking data to make sure users cannot get stuck.
+		opts.MaxRepackBytes = 0
+	}
 
 	maxUnused := strings.TrimSpace(opts.MaxUnused)
 	if maxUnused == "" {
@@ -126,11 +140,31 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 		return err
 	}
 
+	if opts.RepackUncompressed && gopts.Compression == repository.CompressionOff {
+		return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
 	}
 
+	if repo.Backend().Connections() < 2 {
+		return errors.Fatal("prune requires a backend connection limit of at least two")
+	}
+
+	if repo.Config().Version < 2 && opts.RepackUncompressed {
+		return errors.Fatal("compression requires at least repository format version 2")
+	}
+
+	if opts.UnsafeNoSpaceRecovery != "" {
+		repoID := repo.Config().ID
+		if opts.UnsafeNoSpaceRecovery != repoID {
+			return errors.Fatalf("must pass id '%s' to --unsafe-recover-no-free-space", repoID)
+		}
+		opts.unsafeRecovery = true
+	}
+
 	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
@@ -149,26 +183,69 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
 	}
 
 	Verbosef("loading indexes...\n")
+	// loading the index before the snapshots is ok, as we use an exclusive lock here
 	err := repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 
-	usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
+	plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots)
 	if err != nil {
 		return err
 	}
 
-	return prune(opts, gopts, repo, usedBlobs)
+	err = printPruneStats(gopts, stats)
+	if err != nil {
+		return err
+	}
+
+	return doPrune(opts, gopts, repo, plan)
+}
+
+type pruneStats struct {
+	blobs struct {
+		used      uint
+		duplicate uint
+		unused    uint
+		remove    uint
+		repack    uint
+		repackrm  uint
+	}
+	size struct {
+		used      uint64
+		duplicate uint64
+		unused    uint64
+		remove    uint64
+		repack    uint64
+		repackrm  uint64
+		unref     uint64
+	}
+	packs struct {
+		used       uint
+		unused     uint
+		partlyUsed uint
+		unref      uint
+		keep       uint
+		repack     uint
+		remove     uint
+	}
+}
+
+type prunePlan struct {
+	removePacksFirst restic.IDSet   // packs to remove first (unreferenced packs)
+	repackPacks      restic.IDSet   // packs to repack
+	keepBlobs        restic.BlobSet // blobs to keep during repacking
+	removePacks      restic.IDSet   // packs to remove
+	ignorePacks      restic.IDSet   // packs to ignore when rebuilding the index
 }
 
 type packInfo struct {
-	usedBlobs      uint
-	unusedBlobs    uint
-	duplicateBlobs uint
-	usedSize       uint64
-	unusedSize     uint64
-	tpe            restic.BlobType
+	usedBlobs    uint
+	unusedBlobs  uint
+	usedSize     uint64
+	unusedSize   uint64
+	tpe          restic.BlobType
+	uncompressed bool
 }
 
 type packInfoWithID struct {
@@ -176,44 +253,53 @@ type packInfoWithID struct {
 	packInfo
 }
 
-// prune selects which files to rewrite and then does that. The map usedBlobs is
-// modified in the process.
-func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedBlobs restic.BlobSet) error {
+// planPrune selects which files to rewrite and which to delete and which blobs to keep.
+// Also some summary statistics are returned.
+func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) {
 	ctx := gopts.ctx
+	var stats pruneStats
 
-	var stats struct {
-		blobs struct {
-			used      uint
-			duplicate uint
-			unused    uint
-			remove    uint
-			repack    uint
-			repackrm  uint
-		}
-		size struct {
-			used      uint64
-			duplicate uint64
-			unused    uint64
-			remove    uint64
-			repack    uint64
-			repackrm  uint64
-			unref     uint64
-		}
-		packs struct {
-			used       uint
-			unused     uint
-			partlyUsed uint
-			keep       uint
-		}
+	usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
+	if err != nil {
+		return prunePlan{}, stats, err
 	}
 
 	Verbosef("searching used packs...\n")
+	keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats)
+	if err != nil {
+		return prunePlan{}, stats, err
+	}
 
+	Verbosef("collecting packs for deletion and repacking\n")
+	plan, err := decidePackAction(ctx, opts, gopts, repo, indexPack, &stats)
+	if err != nil {
+		return prunePlan{}, stats, err
+	}
+
+	if len(plan.repackPacks) != 0 {
+		// when repacking, we do not want to keep blobs which are
+		// already contained in kept packs, so delete them from keepBlobs
+		for blob := range repo.Index().Each(ctx) {
+			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
+				continue
+			}
+			keepBlobs.Delete(blob.BlobHandle)
+		}
+	} else {
+		// keepBlobs is only needed if packs are repacked
+		keepBlobs = nil
+	}
+	plan.keepBlobs = keepBlobs
+
+	return plan, stats, nil
+}
+
+func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.BlobSet, stats *pruneStats) (restic.BlobSet, map[restic.ID]packInfo, error) {
 	keepBlobs := restic.NewBlobSet()
-	duplicateBlobs := restic.NewBlobSet()
+	duplicateBlobs := make(map[restic.BlobHandle]uint8)
 
 	// iterate over all blobs in index to find out which blobs are duplicates
-	for blob := range repo.Index().Each(ctx) {
+	for blob := range idx.Each(ctx) {
 		bh := blob.BlobHandle
 		size := uint64(blob.Length)
 		switch {
@@ -223,7 +309,16 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			stats.size.used += size
 			stats.blobs.used++
 		case keepBlobs.Has(bh): // duplicate blob
-			duplicateBlobs.Insert(bh)
+			count, ok := duplicateBlobs[bh]
+			if !ok {
+				count = 2 // this one is already the second blob!
+			} else if count < math.MaxUint8 {
+				// don't overflow, but saturate count at 255
+				// this can lead to a non-optimal pack selection, but won't cause
+				// problems otherwise
+				count++
+			}
+			duplicateBlobs[bh] = count
 			stats.size.duplicate += size
 			stats.blobs.duplicate++
 		default:
@@ -239,19 +334,19 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			"Will not start prune to prevent (additional) data loss!\n"+
 			"Please report this error (along with the output of the 'prune' run) at\n"+
 			"https://github.com/restic/restic/issues/new/choose\n", usedBlobs)
-		return errorIndexIncomplete
+		return nil, nil, errorIndexIncomplete
 	}
 
 	indexPack := make(map[restic.ID]packInfo)
 
 	// save computed pack header size
-	for pid, hdrSize := range repo.Index().PackSize(ctx, true) {
+	for pid, hdrSize := range pack.Size(ctx, idx, true) {
 		// initialize tpe with NumBlobTypes to indicate it's not set
 		indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
 	}
 
 	// iterate over all blobs in index to generate packInfo
-	for blob := range repo.Index().Each(ctx) {
+	for blob := range idx.Each(ctx) {
 		ip := indexPack[blob.PackID]
 
 		// Set blob type if not yet set
@@ -266,10 +361,9 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 		bh := blob.BlobHandle
 		size := uint64(blob.Length)
+		_, isDuplicate := duplicateBlobs[bh]
 		switch {
-		case duplicateBlobs.Has(bh): // duplicate blob
-			ip.usedSize += size
-			ip.duplicateBlobs++
+		case isDuplicate: // duplicate blobs will be handled later
 		case keepBlobs.Has(bh): // used blob, not duplicate
 			ip.usedSize += size
 			ip.usedBlobs++
@@ -277,23 +371,66 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			ip.unusedSize += size
 			ip.unusedBlobs++
 		}
+		if !blob.IsCompressed() {
+			ip.uncompressed = true
+		}
 		// update indexPack
 		indexPack[blob.PackID] = ip
 	}
 
-	Verbosef("collecting packs for deletion and repacking\n")
+	// if duplicate blobs exist, those will be set to either "used" or "unused":
+	// - mark only one occurence of duplicate blobs as used
+	// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
+	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
+	if len(duplicateBlobs) > 0 {
+		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
+		for blob := range idx.Each(ctx) {
+			bh := blob.BlobHandle
+			count, isDuplicate := duplicateBlobs[bh]
+			if !isDuplicate {
+				continue
+			}
+
+			ip := indexPack[blob.PackID]
+			size := uint64(blob.Length)
+			switch {
+			case count == 0:
+				// used duplicate exists ->  mark as unused
+				ip.unusedSize += size
+				ip.unusedBlobs++
+			case ip.usedBlobs > 0, count == 1:
+				// other used blobs in pack or "last" occurency ->  mark as used
+				ip.usedSize += size
+				ip.usedBlobs++
+				// let other occurences be marked as unused
+				duplicateBlobs[bh] = 0
+			default:
+				// mark as unused and decrease counter
+				ip.unusedSize += size
+				ip.unusedBlobs++
+				duplicateBlobs[bh] = count - 1
+			}
+			// update indexPack
+			indexPack[blob.PackID] = ip
+		}
+	}
+
+	return keepBlobs, indexPack, nil
+}
+
+func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats) (prunePlan, error) {
 	removePacksFirst := restic.NewIDSet()
 	removePacks := restic.NewIDSet()
 	repackPacks := restic.NewIDSet()
 
 	var repackCandidates []packInfoWithID
-	repackAllPacksWithDuplicates := true
-
-	keep := func(p packInfo) {
-		stats.packs.keep++
-		if p.duplicateBlobs > 0 {
-			repackAllPacksWithDuplicates = false
-		}
+	var repackSmallCandidates []packInfoWithID
+	repoVersion := repo.Config().Version
+	// only repack very small files by default
+	targetPackSize := repo.PackSize() / 25
+	if opts.RepackSmall {
+		// consider files with at least 80% of the target size as large enough
+		targetPackSize = repo.PackSize() / 5 * 4
 	}
 
 	// loop over all packs and decide what to do
@@ -308,8 +445,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			return nil
 		}
 
-		if p.unusedSize+p.usedSize != uint64(packSize) &&
-			!(p.usedBlobs == 0 && p.duplicateBlobs == 0) {
+		if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
 			// Pack size does not fit and pack is needed => error
 			// If the pack is not needed, this is no error, the pack can
 			// and will be simply removed, see below.
@@ -320,7 +456,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 		// statistics
 		switch {
-		case p.usedBlobs == 0 && p.duplicateBlobs == 0:
+		case p.usedBlobs == 0:
 			stats.packs.unused++
 		case p.unusedBlobs == 0:
 			stats.packs.used++
@@ -328,9 +464,18 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			stats.packs.partlyUsed++
 		}
 
+		mustCompress := false
+		if repoVersion >= 2 {
+			// repo v2: always repack tree blobs if uncompressed
+			// compress data blobs if requested
+			mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
+		}
+		// use a flag that pack must be compressed
+		p.uncompressed = mustCompress
+
 		// decide what to do
 		switch {
-		case p.usedBlobs == 0 && p.duplicateBlobs == 0:
+		case p.usedBlobs == 0:
 			// All blobs in pack are no longer used => remove pack!
 			removePacks.Insert(id)
 			stats.blobs.remove += p.unusedBlobs
@@ -338,11 +483,15 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 		case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
 			// if this is a data pack and --repack-cacheable-only is set => keep pack!
-			keep(p)
+			stats.packs.keep++
 
-		case p.unusedBlobs == 0 && p.duplicateBlobs == 0 && p.tpe != restic.InvalidBlob:
-			// All blobs in pack are used and not duplicates/mixed => keep pack!
-			keep(p)
+		case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
+			if packSize >= int64(targetPackSize) {
+				// All blobs in pack are used and not mixed => keep pack!
+				stats.packs.keep++
+			} else {
+				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p})
+			}
 
 		default:
 			// all other packs are candidates for repacking
@@ -355,7 +504,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	})
 	bar.Done()
 	if err != nil {
-		return err
+		return prunePlan{}, err
 	}
 
 	// At this point indexPacks contains only missing packs!
@@ -363,7 +512,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	// missing packs that are not needed can be ignored
 	ignorePacks := restic.NewIDSet()
 	for id, p := range indexPack {
-		if p.usedBlobs == 0 && p.duplicateBlobs == 0 {
+		if p.usedBlobs == 0 {
 			ignorePacks.Insert(id)
 			stats.blobs.remove += p.unusedBlobs
 			stats.size.remove += p.unusedSize
@@ -376,7 +525,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		for id := range indexPack {
 			Warnf("  %v\n", id)
 		}
-		return errorPacksMissing
+		return prunePlan{}, errorPacksMissing
 	}
 	if len(ignorePacks) != 0 {
 		Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
@@ -385,65 +534,81 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		}
 	}
 
-	// calculate limit for number of unused bytes in the repo after repacking
-	maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
+	if len(repackSmallCandidates) < 10 {
+		// too few small files to be worth the trouble, this also prevents endlessly repacking
+		// if there is just a single pack file below the target size
+		stats.packs.keep += uint(len(repackSmallCandidates))
+	} else {
+		repackCandidates = append(repackCandidates, repackSmallCandidates...)
+	}
 
 	// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
 	// This is equivalent to sorting by unused / total space.
 	// Instead of unused[i] / used[i] > unused[j] / used[j] we use
 	// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
-	// Morover duplicates and packs containing trees are sorted to the beginning
+	// Moreover packs containing trees and too small packs are sorted to the beginning
 	sort.Slice(repackCandidates, func(i, j int) bool {
 		pi := repackCandidates[i].packInfo
 		pj := repackCandidates[j].packInfo
 		switch {
-		case pi.duplicateBlobs > 0 && pj.duplicateBlobs == 0:
-			return true
-		case pj.duplicateBlobs > 0 && pi.duplicateBlobs == 0:
-			return false
 		case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
 			return true
 		case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
 			return false
+		case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
+			return true
+		case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
+			return false
 		}
 		return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
 	})
 
 	repack := func(id restic.ID, p packInfo) {
 		repackPacks.Insert(id)
-		stats.blobs.repack += p.unusedBlobs + p.duplicateBlobs + p.usedBlobs
+		stats.blobs.repack += p.unusedBlobs + p.usedBlobs
 		stats.size.repack += p.unusedSize + p.usedSize
 		stats.blobs.repackrm += p.unusedBlobs
 		stats.size.repackrm += p.unusedSize
 	}
 
+	// calculate limit for number of unused bytes in the repo after repacking
+	maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
+
 	for _, p := range repackCandidates {
 		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
 		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
+		packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
 
 		switch {
 		case reachedRepackSize:
-			keep(p.packInfo)
+			stats.packs.keep++
 
-		case p.duplicateBlobs > 0, p.tpe != restic.DataBlob:
-			// repacking duplicates/non-data is only limited by repackSize
+		case p.tpe != restic.DataBlob, p.uncompressed:
+			// repacking non-data packs / uncompressed-trees is only limited by repackSize
 			repack(p.ID, p.packInfo)
 
-		case reachedUnusedSizeAfter:
+		case reachedUnusedSizeAfter && packIsLargeEnough:
 			// for all other packs stop repacking if tolerated unused size is reached.
-			keep(p.packInfo)
+			stats.packs.keep++
 
 		default:
 			repack(p.ID, p.packInfo)
 		}
 	}
 
-	// if all duplicates are repacked, print out correct statistics
-	if repackAllPacksWithDuplicates {
-		stats.blobs.repackrm += stats.blobs.duplicate
-		stats.size.repackrm += stats.size.duplicate
-	}
+	stats.packs.unref = uint(len(removePacksFirst))
+	stats.packs.repack = uint(len(repackPacks))
+	stats.packs.remove = uint(len(removePacks))
 
+	return prunePlan{removePacksFirst: removePacksFirst,
+		removePacks: removePacks,
+		repackPacks: repackPacks,
+		ignorePacks: ignorePacks,
+	}, nil
+}
+
+// printPruneStats prints out the statistics
+func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
 	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
 	if stats.blobs.duplicate > 0 {
 		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
@@ -473,73 +638,109 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
 
 	Verboseff("to keep:      %10d packs\n", stats.packs.keep)
-	Verboseff("to repack:    %10d packs\n", len(repackPacks))
-	Verboseff("to delete:    %10d packs\n", len(removePacks))
-	if len(removePacksFirst) > 0 {
-		Verboseff("to delete:    %10d unreferenced packs\n\n", len(removePacksFirst))
+	Verboseff("to repack:    %10d packs\n", stats.packs.repack)
+	Verboseff("to delete:    %10d packs\n", stats.packs.remove)
+	if stats.packs.unref > 0 {
+		Verboseff("to delete:    %10d unreferenced packs\n\n", stats.packs.unref)
 	}
+	return nil
+}
+
+// doPrune does the actual pruning:
+// - remove unreferenced packs first
+// - repack given pack files while keeping the given blobs
+// - rebuild the index while ignoring all files that will be deleted
+// - delete the files
+// plan.removePacks and plan.ignorePacks are modified in this function.
+func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
+	ctx := gopts.ctx
 
 	if opts.DryRun {
 		if !gopts.JSON && gopts.verbosity >= 2 {
-			if len(removePacksFirst) > 0 {
-				Printf("Would have removed the following unreferenced packs:\n%v\n\n", removePacksFirst)
+			if len(plan.removePacksFirst) > 0 {
+				Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
 			}
-			Printf("Would have repacked and removed the following packs:\n%v\n\n", repackPacks)
-			Printf("Would have removed the following no longer used packs:\n%v\n\n", removePacks)
+			Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
+			Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
 		}
 		// Always quit here if DryRun was set!
 		return nil
 	}
 
 	// unreferenced packs can be safely deleted first
-	if len(removePacksFirst) != 0 {
+	if len(plan.removePacksFirst) != 0 {
 		Verbosef("deleting unreferenced packs\n")
-		DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile)
+		DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile)
 	}
 
-	if len(repackPacks) != 0 {
+	if len(plan.repackPacks) != 0 {
 		Verbosef("repacking packs\n")
-		bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked")
-		_, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar)
+		bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked")
+		_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
 		bar.Done()
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
 
 		// Also remove repacked packs
-		removePacks.Merge(repackPacks)
+		plan.removePacks.Merge(plan.repackPacks)
+
+		if len(plan.keepBlobs) != 0 {
+			Warnf("%v was not repacked\n\n"+
+				"Integrity check failed.\n"+
+				"Please report this error (along with the output of the 'prune' run) at\n"+
+				"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
+			return errors.Fatal("internal error: blobs were not repacked")
+		}
 	}
 
-	if len(ignorePacks) == 0 {
-		ignorePacks = removePacks
+	if len(plan.ignorePacks) == 0 {
+		plan.ignorePacks = plan.removePacks
 	} else {
-		ignorePacks.Merge(removePacks)
+		plan.ignorePacks.Merge(plan.removePacks)
 	}
 
-	if len(ignorePacks) != 0 {
-		err = rebuildIndexFiles(gopts, repo, ignorePacks, nil)
+	if opts.unsafeRecovery {
+		Verbosef("deleting index files\n")
+		indexFiles := repo.Index().(*repository.MasterIndex).IDs()
+		err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
+	} else if len(plan.ignorePacks) != 0 {
+		err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil)
 		if err != nil {
 			return errors.Fatalf("%s", err)
 		}
 	}
 
-	if len(removePacks) != 0 {
-		Verbosef("removing %d old packs\n", len(removePacks))
-		DeleteFiles(gopts, repo, removePacks, restic.PackFile)
+	if len(plan.removePacks) != 0 {
+		Verbosef("removing %d old packs\n", len(plan.removePacks))
+		DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile)
+	}
+
+	if opts.unsafeRecovery {
+		_, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
 	}
 
 	Verbosef("done\n")
 	return nil
 }
 
-func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
+func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
 	Verbosef("rebuilding index\n")
 
-	idx := (repo.Index()).(*repository.MasterIndex)
-	packcount := uint64(len(idx.Packs(removePacks)))
-	bar := newProgressMax(!gopts.Quiet, packcount, "packs processed")
-	obsoleteIndexes, err := idx.Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
+	bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
+	obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
 	bar.Done()
+	return obsoleteIndexes, err
+}
+
+func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
+	obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete)
 	if err != nil {
 		return err
 	}
@@ -553,17 +754,18 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r
 
 	var snapshotTrees restic.IDs
 	Verbosef("loading all snapshots...\n")
-	err = restic.ForAllSnapshots(gopts.ctx, repo, ignoreSnapshots,
+	err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots,
 		func(id restic.ID, sn *restic.Snapshot, err error) error {
-			debug.Log("add snapshot %v (tree %v, error %v)", id, *sn.Tree, err)
 			if err != nil {
+				debug.Log("failed to load snapshot %v (error %v)", id, err)
 				return err
 			}
+			debug.Log("add snapshot %v (tree %v)", id, *sn.Tree)
 			snapshotTrees = append(snapshotTrees, *sn.Tree)
 			return nil
 		})
 	if err != nil {
-		return nil, err
+		return nil, errors.Fatalf("failed loading snapshot: %v", err)
 	}
 
 	Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
@@ -576,7 +778,7 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r
 	err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
 	if err != nil {
 		if repo.Backend().IsNotExist(err) {
-			return nil, errors.Fatal("unable to load a tree from the repo: " + err.Error())
+			return nil, errors.Fatal("unable to load a tree from the repository: " + err.Error())
 		}
 
 		return nil, err

cmd/restic/cmd_rebuild_index.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 
@@ -97,7 +98,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		if err != nil {
 			return err
 		}
-		packSizeFromIndex = repo.Index().PackSize(ctx, false)
+		packSizeFromIndex = pack.Size(ctx, repo.Index(), false)
 	}
 
 	Verbosef("getting pack files to read...\n")

cmd/restic/cmd_recover.go

@@ -5,9 +5,11 @@ import (
 	"os"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
 )
 
 var cmdRecover = &cobra.Command{
@@ -50,6 +52,11 @@ func runRecover(gopts GlobalOptions) error {
 		return err
 	}
 
+	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
 	Verbosef("load index files\n")
 	if err = repo.LoadIndex(gopts.ctx); err != nil {
 		return err
@@ -68,7 +75,7 @@ func runRecover(gopts GlobalOptions) error {
 	Verbosef("load %d trees\n", len(trees))
 	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
-		tree, err := repo.LoadTree(gopts.ctx, id)
+		tree, err := restic.LoadTree(gopts.ctx, repo, id)
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
 			continue
@@ -84,7 +91,7 @@ func runRecover(gopts GlobalOptions) error {
 	bar.Done()
 
 	Verbosef("load snapshots\n")
-	err = restic.ForAllSnapshots(gopts.ctx, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+	err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
 		trees[*sn.Tree] = true
 		return nil
 	})
@@ -125,14 +132,26 @@ func runRecover(gopts GlobalOptions) error {
 		}
 	}
 
-	treeID, err := repo.SaveTree(gopts.ctx, tree)
-	if err != nil {
-		return errors.Fatalf("unable to save new tree to the repo: %v", err)
-	}
+	wg, ctx := errgroup.WithContext(gopts.ctx)
+	repo.StartPackUploader(ctx, wg)
 
-	err = repo.Flush(gopts.ctx)
+	var treeID restic.ID
+	wg.Go(func() error {
+		var err error
+		treeID, err = restic.SaveTree(ctx, repo, tree)
+		if err != nil {
+			return errors.Fatalf("unable to save new tree to the repository: %v", err)
+		}
+
+		err = repo.Flush(ctx)
+		if err != nil {
+			return errors.Fatalf("unable to save blobs to the repository: %v", err)
+		}
+		return nil
+	})
+	err = wg.Wait()
 	if err != nil {
-		return errors.Fatalf("unable to save blobs to the repo: %v", err)
+		return err
 	}
 
 	return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
@@ -147,7 +166,7 @@ func createSnapshot(ctx context.Context, name, hostname string, tags []string, r
 
 	sn.Tree = tree
 
-	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+	id, err := restic.SaveSnapshot(ctx, repo, sn)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}

cmd/restic/cmd_restore.go

@@ -70,6 +70,28 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0
 	hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0
 
+	// Validate provided patterns
+	if len(opts.Exclude) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.Exclude); !valid {
+			return errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+	}
+	if len(opts.InsensitiveExclude) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExclude); !valid {
+			return errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+	}
+	if len(opts.Include) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.Include); !valid {
+			return errors.Fatalf("--include: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+	}
+	if len(opts.InsensitiveInclude) > 0 {
+		if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveInclude); !valid {
+			return errors.Fatalf("--iinclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+		}
+	}
+
 	for i, str := range opts.InsensitiveExclude {
 		opts.InsensitiveExclude[i] = strings.ToLower(str)
 	}
@@ -110,25 +132,25 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex(ctx)
-	if err != nil {
-		return err
-	}
-
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
+		id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
-		id, err = restic.FindSnapshot(ctx, repo, snapshotIDString)
+		id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString)
 		if err != nil {
 			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
 		}
 	}
 
+	err = repo.LoadIndex(ctx)
+	if err != nil {
+		return err
+	}
+
 	res, err := restorer.NewRestorer(ctx, repo, id)
 	if err != nil {
 		Exitf(2, "creating restorer failed: %v\n", err)

cmd/restic/cmd_snapshots.go

@@ -58,7 +58,7 @@ func init() {
 		panic(err)
 	}
 	f.IntVar(&snapshotOptions.Latest, "latest", 0, "only show the last `n` snapshots for each host and path")
-	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
+	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma")
 }
 
 func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
@@ -79,7 +79,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	defer cancel()
 
 	var snapshots restic.Snapshots
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		snapshots = append(snapshots, sn)
 	}
 	snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)

cmd/restic/cmd_stats.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/walker"
 
@@ -86,10 +87,6 @@ func runStats(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if err = repo.LoadIndex(ctx); err != nil {
-		return err
-	}
-
 	if !gopts.NoLock {
 		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
@@ -98,6 +95,15 @@ func runStats(gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
+	if err = repo.LoadIndex(ctx); err != nil {
+		return err
+	}
+
 	if !gopts.JSON {
 		Printf("scanning...\n")
 	}
@@ -105,13 +111,12 @@ func runStats(gopts GlobalOptions, args []string) error {
 	// create a container for the stats (and other needed state)
 	stats := &statsContainer{
 		uniqueFiles:    make(map[fileID]struct{}),
-		uniqueInodes:   make(map[uint64]struct{}),
 		fileBlobs:      make(map[string]restic.IDSet),
 		blobs:          restic.NewBlobSet(),
-		snapshotsCount: 0,
+		SnapshotsCount: 0,
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) {
 		err = statsWalkSnapshot(ctx, sn, repo, stats)
 		if err != nil {
 			return fmt.Errorf("error walking snapshot: %v", err)
@@ -125,11 +130,11 @@ func runStats(gopts GlobalOptions, args []string) error {
 	if statsOptions.countMode == countModeRawData {
 		// the blob handles have been collected, but not yet counted
 		for blobHandle := range stats.blobs {
-			blobSize, found := repo.LookupBlobSize(blobHandle.ID, blobHandle.Type)
-			if !found {
+			pbs := repo.Index().Lookup(blobHandle)
+			if len(pbs) == 0 {
 				return fmt.Errorf("blob %v not found", blobHandle)
 			}
-			stats.TotalSize += uint64(blobSize)
+			stats.TotalSize += uint64(pbs[0].Length)
 			stats.TotalBlobCount++
 		}
 	}
@@ -143,7 +148,7 @@ func runStats(gopts GlobalOptions, args []string) error {
 	}
 
 	Printf("Stats in %s mode:\n", statsOptions.countMode)
-	Printf("Snapshots processed:   %d\n", stats.snapshotsCount)
+	Printf("Snapshots processed:   %d\n", stats.SnapshotsCount)
 
 	if stats.TotalBlobCount > 0 {
 		Printf("   Total Blob Count:   %d\n", stats.TotalBlobCount)
@@ -161,7 +166,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
 	}
 
-	stats.snapshotsCount++
+	stats.SnapshotsCount++
 
 	if statsOptions.countMode == countModeRawData {
 		// count just the sizes of unique blobs; we don't need to walk the tree
@@ -169,7 +174,8 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 		return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
 	}
 
-	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats))
+	uniqueInodes := make(map[uint64]struct{})
+	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats, uniqueInodes))
 	if err != nil {
 		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
 	}
@@ -177,7 +183,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 	return nil
 }
 
-func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFunc {
+func statsWalkTree(repo restic.Repository, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc {
 	return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
 		if nodeErr != nil {
 			return true, nodeErr
@@ -236,8 +242,8 @@ func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFun
 
 			// if inodes are present, only count each inode once
 			// (hard links do not increase restore size)
-			if _, ok := stats.uniqueInodes[node.Inode]; !ok || node.Inode == 0 {
-				stats.uniqueInodes[node.Inode] = struct{}{}
+			if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 {
+				uniqueInodes[node.Inode] = struct{}{}
 				stats.TotalSize += node.Size
 			}
 
@@ -279,15 +285,13 @@ type statsContainer struct {
 	TotalSize      uint64 `json:"total_size"`
 	TotalFileCount uint64 `json:"total_file_count"`
 	TotalBlobCount uint64 `json:"total_blob_count,omitempty"`
+	// holds count of all considered snapshots
+	SnapshotsCount int `json:"snapshots_count"`
 
 	// uniqueFiles marks visited files according to their
 	// contents (hashed sequence of content blob IDs)
 	uniqueFiles map[fileID]struct{}
 
-	// uniqueInodes marks visited files according to their
-	// inode # (hashed sequence of inode numbers)
-	uniqueInodes map[uint64]struct{}
-
 	// fileBlobs maps a file name (path) to the set of
 	// blobs that have been seen as a part of the file
 	fileBlobs map[string]restic.IDSet
@@ -295,9 +299,6 @@ type statsContainer struct {
 	// blobs is used to count individual unique blobs,
 	// independent of references to files
 	blobs restic.BlobSet
-
-	// holds count of all considered snapshots
-	snapshotsCount int
 }
 
 // fileID is a 256-bit hash that distinguishes unique files.

cmd/restic/cmd_tag.go

@@ -82,17 +82,13 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
 		}
 
 		// Save the new snapshot.
-		id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+		id, err := restic.SaveSnapshot(ctx, repo, sn)
 		if err != nil {
 			return false, err
 		}
 
 		debug.Log("new snapshot saved as %v", id)
 
-		if err = repo.Flush(ctx); err != nil {
-			return false, err
-		}
-
 		// Remove the old snapshot.
 		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
 		if err = repo.Backend().Remove(ctx, h); err != nil {
@@ -129,7 +125,7 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
 	changeCnt := 0
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
 		if err != nil {
 			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)

cmd/restic/delete.go

@@ -18,8 +18,6 @@ func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList re
 	return deleteFiles(gopts, false, repo, fileList, fileType)
 }
 
-const numDeleteWorkers = 8
-
 // deleteFiles deletes the given fileList of fileType in parallel
 // if ignoreError=true, it will print a warning if there was an error, else it will abort.
 func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
@@ -40,7 +38,9 @@ func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository,
 
 	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted")
 	defer bar.Done()
-	for i := 0; i < numDeleteWorkers; i++ {
+	// deleting files is IO-bound
+	workerCount := repo.Connections()
+	for i := 0; i < int(workerCount); i++ {
 		wg.Go(func() error {
 			for id := range fileChan {
 				h := restic.Handle{Type: fileType, Name: id.String()}

cmd/restic/find.go

@@ -3,33 +3,39 @@ package main
 import (
 	"context"
 
-	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/restic"
 )
 
 // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
-func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
+func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
 	out := make(chan *restic.Snapshot)
 	go func() {
 		defer close(out)
 		if len(snapshotIDs) != 0 {
+			// memorize snapshots list to prevent repeated backend listings
+			be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
+			if err != nil {
+				Warnf("could not load snapshots: %v\n", err)
+				return
+			}
+
 			var (
 				id         restic.ID
 				usedFilter bool
-				err        error
 			)
 			ids := make(restic.IDs, 0, len(snapshotIDs))
 			// Process all snapshot IDs given as arguments.
 			for _, s := range snapshotIDs {
 				if s == "latest" {
 					usedFilter = true
-					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts, nil)
+					id, err = restic.FindLatestSnapshot(ctx, be, loader, paths, tags, hosts, nil)
 					if err != nil {
 						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
 						continue
 					}
 				} else {
-					id, err = restic.FindSnapshot(ctx, repo, s)
+					id, err = restic.FindSnapshot(ctx, be, s)
 					if err != nil {
 						Warnf("Ignoring %q: %v\n", s, err)
 						continue
@@ -44,7 +50,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			}
 
 			for _, id := range ids.Uniq() {
-				sn, err := restic.LoadSnapshot(ctx, repo, id)
+				sn, err := restic.LoadSnapshot(ctx, loader, id)
 				if err != nil {
 					Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
 					continue
@@ -58,7 +64,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			return
 		}
 
-		snapshots, err := restic.FindFilteredSnapshots(ctx, repo, hosts, tags, paths)
+		snapshots, err := restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths)
 		if err != nil {
 			Warnf("could not load snapshots: %v\n", err)
 			return

cmd/restic/global.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -16,6 +17,7 @@ import (
 	"github.com/restic/restic/internal/backend/azure"
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/gs"
+	"github.com/restic/restic/internal/backend/limiter"
 	"github.com/restic/restic/internal/backend/local"
 	"github.com/restic/restic/internal/backend/location"
 	"github.com/restic/restic/internal/backend/rclone"
@@ -26,7 +28,6 @@ import (
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/fs"
-	"github.com/restic/restic/internal/limiter"
 	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -37,10 +38,10 @@ import (
 
 	"os/exec"
 
-	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/term"
 )
 
-var version = "0.13.0"
+var version = "0.14.0"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -60,13 +61,12 @@ type GlobalOptions struct {
 	JSON            bool
 	CacheDir        string
 	NoCache         bool
-	CACerts         []string
-	InsecureTLS     bool
-	TLSClientCert   string
 	CleanupCache    bool
+	Compression     repository.CompressionMode
+	PackSize        uint
 
-	LimitUploadKb   int
-	LimitDownloadKb int
+	backend.TransportOptions
+	limiter.Limits
 
 	ctx      context.Context
 	password string
@@ -104,6 +104,9 @@ func init() {
 		return nil
 	})
 
+	// parse target pack size from env, on error the default value will be used
+	targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
+
 	f := cmdRoot.PersistentFlags()
 	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", os.Getenv("RESTIC_REPOSITORY_FILE"), "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
@@ -116,16 +119,24 @@ func init() {
 	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
 	f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)")
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
-	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "`file` to load root certificates from (default: use system certificates)")
-	f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key")
-	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repo (insecure)")
+	f.StringSliceVar(&globalOptions.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates)")
+	f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key")
+	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
-	f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
-	f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
+	f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)")
+	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
+	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
+	f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
 	// Use our "generate" command instead of the cobra provided "completion" command
 	cmdRoot.CompletionOptions.DisableDefaultCmd = true
 
+	comp := os.Getenv("RESTIC_COMPRESSION")
+	if comp != "" {
+		// ignore error as there's no good way to handle it
+		_ = globalOptions.Compression.Set(comp)
+	}
+
 	restoreTerminal()
 }
 
@@ -145,13 +156,13 @@ func checkErrno(err error) error {
 }
 
 func stdinIsTerminal() bool {
-	return terminal.IsTerminal(int(os.Stdin.Fd()))
+	return term.IsTerminal(int(os.Stdin.Fd()))
 }
 
 func stdoutIsTerminal() bool {
 	// mintty on windows can use pipes which behave like a posix terminal,
 	// but which are not a terminal handle
-	return terminal.IsTerminal(int(os.Stdout.Fd())) || stdoutCanUpdateStatus()
+	return term.IsTerminal(int(os.Stdout.Fd())) || stdoutCanUpdateStatus()
 }
 
 func stdoutCanUpdateStatus() bool {
@@ -159,7 +170,7 @@ func stdoutCanUpdateStatus() bool {
 }
 
 func stdoutTerminalWidth() int {
-	w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+	w, _, err := term.GetSize(int(os.Stdout.Fd()))
 	if err != nil {
 		return 0
 	}
@@ -172,12 +183,12 @@ func stdoutTerminalWidth() int {
 // program execution must revert changes to the terminal configuration itself.
 // The terminal configuration is only restored while reading a password.
 func restoreTerminal() {
-	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
+	if !term.IsTerminal(int(os.Stdout.Fd())) {
 		return
 	}
 
 	fd := int(os.Stdout.Fd())
-	state, err := terminal.GetState(fd)
+	state, err := term.GetState(fd)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
 		return
@@ -192,7 +203,7 @@ func restoreTerminal() {
 		if !isReadingPassword {
 			return nil
 		}
-		err := checkErrno(terminal.Restore(fd, state))
+		err := checkErrno(term.Restore(fd, state))
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
 		}
@@ -295,7 +306,7 @@ func resolvePassword(opts GlobalOptions, envStr string) (string, error) {
 	}
 	if opts.PasswordFile != "" {
 		s, err := textfile.Read(opts.PasswordFile)
-		if os.IsNotExist(errors.Cause(err)) {
+		if errors.Is(err, os.ErrNotExist) {
 			return "", errors.Fatalf("%s does not exist", opts.PasswordFile)
 		}
 		return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
@@ -322,7 +333,7 @@ func readPassword(in io.Reader) (password string, err error) {
 func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) {
 	fmt.Fprint(out, prompt)
 	isReadingPassword = true
-	buf, err := terminal.ReadPassword(int(in.Fd()))
+	buf, err := term.ReadPassword(int(in.Fd()))
 	isReadingPassword = false
 	fmt.Fprintln(out)
 	if err != nil {
@@ -396,7 +407,7 @@ func ReadRepo(opts GlobalOptions) (string, error) {
 		}
 
 		s, err := textfile.Read(opts.RepositoryFile)
-		if os.IsNotExist(errors.Cause(err)) {
+		if errors.Is(err, os.ErrNotExist) {
 			return "", errors.Fatalf("%s does not exist", opts.RepositoryFile)
 		}
 		if err != nil {
@@ -435,7 +446,13 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		}
 	}
 
-	s := repository.New(be)
+	s, err := repository.New(be, repository.Options{
+		Compression: opts.Compression,
+		PackSize:    opts.PackSize * 1024 * 1024,
+	})
+	if err != nil {
+		return nil, err
+	}
 
 	passwordTriesLeft := 1
 	if stdinIsTerminal() && opts.password == "" {
@@ -455,7 +472,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		err = s.SearchKey(opts.ctx, opts.password, maxKeys, opts.KeyHint)
 		if err != nil && passwordTriesLeft > 1 {
 			opts.password = ""
-			fmt.Printf("%s. Try again\n", err)
+			fmt.Fprintf(os.Stderr, "%s. Try again\n", err)
 		}
 	}
 	if err != nil {
@@ -471,7 +488,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 			id = id[:8]
 		}
 		if !opts.JSON {
-			Verbosef("repository %v opened successfully, password is correct\n", id)
+			Verbosef("repository %v opened (repository version %v) successfully, password is correct\n", id, s.Config().Version)
 		}
 	}
 
@@ -553,13 +570,13 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
 		}
 
-		if cfg.Secret == "" {
-			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
+		if cfg.Secret.String() == "" {
+			cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY"))
 		}
 
-		if cfg.KeyID == "" && cfg.Secret != "" {
+		if cfg.KeyID == "" && cfg.Secret.String() != "" {
 			return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
-		} else if cfg.KeyID != "" && cfg.Secret == "" {
+		} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
 			return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
 		}
 
@@ -593,8 +610,12 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
 		}
 
-		if cfg.AccountKey == "" {
-			cfg.AccountKey = os.Getenv("AZURE_ACCOUNT_KEY")
+		if cfg.AccountKey.String() == "" {
+			cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY"))
+		}
+
+		if cfg.AccountSAS.String() == "" {
+			cfg.AccountSAS = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_SAS"))
 		}
 
 		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
@@ -629,11 +650,11 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
 		}
 
-		if cfg.Key == "" {
-			cfg.Key = os.Getenv("B2_ACCOUNT_KEY")
+		if cfg.Key.String() == "" {
+			cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY"))
 		}
 
-		if cfg.Key == "" {
+		if cfg.Key.String() == "" {
 			return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
 		}
 
@@ -679,18 +700,13 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 		return nil, err
 	}
 
-	tropts := backend.TransportOptions{
-		RootCertFilenames:        globalOptions.CACerts,
-		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
-		InsecureTLS:              globalOptions.InsecureTLS,
-	}
-	rt, err := backend.Transport(tropts)
+	rt, err := backend.Transport(globalOptions.TransportOptions)
 	if err != nil {
 		return nil, err
 	}
 
 	// wrap the transport so that the throughput via HTTP is limited
-	lim := limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb)
+	lim := limiter.NewStaticLimiter(gopts.Limits)
 	rt = lim.Transport(rt)
 
 	switch loc.Scheme {
@@ -718,7 +734,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	}
 
 	if err != nil {
-		return nil, errors.Fatalf("unable to open repo at %v: %v", location.StripPassword(s), err)
+		return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err)
 	}
 
 	// wrap backend if a test specified an inner hook
@@ -760,12 +776,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 		return nil, err
 	}
 
-	tropts := backend.TransportOptions{
-		RootCertFilenames:        globalOptions.CACerts,
-		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
-		InsecureTLS:              globalOptions.InsecureTLS,
-	}
-	rt, err := backend.Transport(tropts)
+	rt, err := backend.Transport(globalOptions.TransportOptions)
 	if err != nil {
 		return nil, err
 	}

cmd/restic/global_debug.go

@@ -1,3 +1,4 @@
+//go:build debug || profile
 // +build debug profile
 
 package main

cmd/restic/global_release.go

@@ -1,3 +1,4 @@
+//go:build !debug && !profile
 // +build !debug,!profile
 
 package main

cmd/restic/integration_filter_pattern_test.go

@@ -0,0 +1,106 @@
+//go:build go1.16
+// +build go1.16
+
+// Before Go 1.16 filepath.Match returned early on a failed match,
+// and thus did not report any later syntax error in the pattern.
+// https://go.dev/doc/go1.16#path/filepath
+
+package main
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	var err error
+
+	// Test --exclude
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+
+	// Test --iexclude
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+}
+
+func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	// Create an exclude file with some invalid patterns
+	excludeFile := env.base + "/excludefile"
+	fileErr := ioutil.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
+	if fileErr != nil {
+		t.Fatalf("Could not write exclude file: %v", fileErr)
+	}
+
+	var err error
+
+	// Test --exclude-file:
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludeFiles: []string{excludeFile}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+
+	// Test --iexclude-file
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludeFiles: []string{excludeFile}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+}
+
+func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+
+	var err error
+
+	// Test --exclude
+	err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{Exclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+
+	// Test --iexclude
+	err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{InsensitiveExclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+
+	// Test --include
+	err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{Include: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+
+	// Test --iinclude
+	err = testRunRestoreAssumeFailure(t, "latest", RestoreOptions{InsensitiveInclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+
+	rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided:
+*[._]log[.-][0-9]
+!*[._]log[.-][0-9]`, err.Error())
+}

cmd/restic/integration_fuse_test.go

@@ -1,3 +1,4 @@
+//go:build darwin || freebsd || linux
 // +build darwin freebsd linux
 
 package main
@@ -54,7 +55,7 @@ func waitForMount(t testing.TB, dir string) {
 
 func testRunMount(t testing.TB, gopts GlobalOptions, dir string) {
 	opts := MountOptions{
-		SnapshotTemplate: time.RFC3339,
+		TimeTemplate: time.RFC3339,
 	}
 	rtest.OK(t, runMount(opts, gopts, []string{dir}))
 }
@@ -153,6 +154,8 @@ func TestMount(t *testing.T) {
 	}
 
 	env, cleanup := withTestEnvironment(t)
+	// must list snapshots more than once
+	env.gopts.backendTestHook = nil
 	defer cleanup()
 
 	testRunInit(t, env.gopts)
@@ -196,6 +199,8 @@ func TestMountSameTimestamps(t *testing.T) {
 	}
 
 	env, cleanup := withTestEnvironment(t)
+	// must list snapshots more than once
+	env.gopts.backendTestHook = nil
 	defer cleanup()
 
 	rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))

cmd/restic/integration_helpers_test.go

@@ -198,6 +198,9 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 		stdout:   os.Stdout,
 		stderr:   os.Stderr,
 		extended: make(options.Options),
+
+		// replace this hook with "nil" if listing a filetype more than once is necessary
+		backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil },
 	}
 
 	// always overwrite global options

cmd/restic/integration_helpers_unix_test.go

@@ -1,4 +1,5 @@
-//+build !windows
+//go:build !windows
+// +build !windows
 
 package main
 

cmd/restic/integration_helpers_windows_test.go

@@ -1,4 +1,5 @@
-//+build windows
+//go:build windows
+// +build windows
 
 package main
 

cmd/restic/integration_test.go

@@ -131,6 +131,12 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
 	rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()}))
 }
 
+func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
+	err := runRestore(opts, gopts, []string{snapshotID})
+
+	return err
+}
+
 func testRunCheck(t testing.TB, gopts GlobalOptions) {
 	opts := CheckOptions{
 		ReadData:    true,
@@ -275,6 +281,11 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
 }
 
 func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
+	oldHook := gopts.backendTestHook
+	gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	defer func() {
+		gopts.backendTestHook = oldHook
+	}()
 	rtest.OK(t, runPrune(opts, gopts))
 }
 
@@ -738,14 +749,17 @@ func TestBackupTags(t *testing.T) {
 }
 
 func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
+	gopts := srcGopts
+	gopts.Repo = dstGopts.Repo
+	gopts.password = dstGopts.password
 	copyOpts := CopyOptions{
 		secondaryRepoOptions: secondaryRepoOptions{
-			Repo:     dstGopts.Repo,
-			password: dstGopts.password,
+			Repo:     srcGopts.Repo,
+			password: srcGopts.password,
 		},
 	}
 
-	rtest.OK(t, runCopy(copyOpts, srcGopts, nil))
+	rtest.OK(t, runCopy(copyOpts, gopts, nil))
 }
 
 func TestCopy(t *testing.T) {
@@ -1035,7 +1049,7 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
 
 	repo, err := OpenRepository(gopts)
 	rtest.OK(t, err)
-	key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 1, "")
+	key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 2, "")
 	rtest.OK(t, err)
 
 	rtest.Equals(t, "john", key.Username)
@@ -1065,6 +1079,8 @@ func TestKeyAddRemove(t *testing.T) {
 	}
 
 	env, cleanup := withTestEnvironment(t)
+	// must list keys more than once
+	env.gopts.backendTestHook = nil
 	defer cleanup()
 
 	testRunInit(t, env.gopts)
@@ -1463,7 +1479,7 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
 	defer func() {
 		repository.IndexFull = indexFull
 	}()
-	repository.IndexFull = func(*repository.Index) bool { return true }
+	repository.IndexFull = func(*repository.Index, bool) bool { return true }
 	testRebuildIndex(t, nil)
 }
 
@@ -1566,29 +1582,43 @@ func TestCheckRestoreNoLock(t *testing.T) {
 }
 
 func TestPrune(t *testing.T) {
-	t.Run("0", func(t *testing.T) {
-		opts := PruneOptions{MaxUnused: "0%"}
+	testPruneVariants(t, false)
+	testPruneVariants(t, true)
+}
+
+func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
+	suffix := ""
+	if unsafeNoSpaceRecovery {
+		suffix = "-recovery"
+	}
+	t.Run("0"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
 		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
 		testPrune(t, opts, checkOpts)
 	})
 
-	t.Run("50", func(t *testing.T) {
-		opts := PruneOptions{MaxUnused: "50%"}
+	t.Run("50"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "50%", unsafeRecovery: unsafeNoSpaceRecovery}
 		checkOpts := CheckOptions{ReadData: true}
 		testPrune(t, opts, checkOpts)
 	})
 
-	t.Run("unlimited", func(t *testing.T) {
-		opts := PruneOptions{MaxUnused: "unlimited"}
+	t.Run("unlimited"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "unlimited", unsafeRecovery: unsafeNoSpaceRecovery}
 		checkOpts := CheckOptions{ReadData: true}
 		testPrune(t, opts, checkOpts)
 	})
 
-	t.Run("CachableOnly", func(t *testing.T) {
-		opts := PruneOptions{MaxUnused: "5%", RepackCachableOnly: true}
+	t.Run("CachableOnly"+suffix, func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "5%", RepackCachableOnly: true, unsafeRecovery: unsafeNoSpaceRecovery}
 		checkOpts := CheckOptions{ReadData: true}
 		testPrune(t, opts, checkOpts)
 	})
+	t.Run("Small", func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "unlimited", RepackSmall: true}
+		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
+		testPrune(t, opts, checkOpts)
+	})
 }
 
 func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
@@ -1659,6 +1689,11 @@ func TestPruneWithDamagedRepository(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 1,
 		"expected one snapshot, got %v", snapshotIDs)
 
+	oldHook := env.gopts.backendTestHook
+	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	defer func() {
+		env.gopts.backendTestHook = oldHook
+	}()
 	// prune should fail
 	rtest.Assert(t, runPrune(pruneDefaultOptions, env.gopts) == errorPacksMissing,
 		"prune should have reported index not complete error")
@@ -1752,12 +1787,22 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 type listOnceBackend struct {
 	restic.Backend
 	listedFileType map[restic.FileType]bool
+	strictOrder    bool
 }
 
 func newListOnceBackend(be restic.Backend) *listOnceBackend {
 	return &listOnceBackend{
 		Backend:        be,
 		listedFileType: make(map[restic.FileType]bool),
+		strictOrder:    false,
+	}
+}
+
+func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend {
+	return &listOnceBackend{
+		Backend:        be,
+		listedFileType: make(map[restic.FileType]bool),
+		strictOrder:    true,
 	}
 }
 
@@ -1765,6 +1810,9 @@ func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(
 	if t != restic.LockFile && be.listedFileType[t] {
 		return errors.Errorf("tried listing type %v the second time", t)
 	}
+	if be.strictOrder && t == restic.SnapshotFile && be.listedFileType[restic.IndexFile] {
+		return errors.Errorf("tried listing type snapshots after index")
+	}
 	be.listedFileType[t] = true
 	return be.Backend.List(ctx, t, fn)
 }
@@ -2135,7 +2183,38 @@ func TestBackendLoadWriteTo(t *testing.T) {
 	firstSnapshot := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(firstSnapshot) == 1,
 		"expected one snapshot, got %v", firstSnapshot)
-
-	// test readData using the hashing.Reader
-	testRunCheck(t, env.gopts)
+}
+
+func TestFindListOnce(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+		return newListOnceBackend(r), nil
+	}
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
+	secondSnapshot := testRunList(t, "snapshots", env.gopts)
+	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
+	thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...)
+
+	repo, err := OpenRepository(env.gopts)
+	rtest.OK(t, err)
+
+	snapshotIDs := restic.NewIDSet()
+	// specify the two oldest snapshots explicitly and use "latest" to reference the newest one
+	for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, nil, nil, nil, []string{
+		secondSnapshot[0].String(),
+		secondSnapshot[1].String()[:8],
+		"latest",
+	}) {
+		snapshotIDs.Insert(*sn.ID())
+	}
+
+	// the snapshots can only be listed once, if both lists match then the there has been only a single List() call
+	rtest.Equals(t, thirdSnapshot, snapshotIDs)
 }

cmd/restic/main.go

@@ -98,11 +98,11 @@ func main() {
 	err := cmdRoot.Execute()
 
 	switch {
-	case restic.IsAlreadyLocked(errors.Cause(err)):
+	case restic.IsAlreadyLocked(err):
 		fmt.Fprintf(os.Stderr, "%v\nthe `unlock` command can be used to remove stale locks\n", err)
 	case err == ErrInvalidSourceData:
 		fmt.Fprintf(os.Stderr, "Warning: %v\n", err)
-	case errors.IsFatal(errors.Cause(err)):
+	case errors.IsFatal(err):
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 	case err != nil:
 		fmt.Fprintf(os.Stderr, "%+v\n", err)

cmd/restic/progress.go

@@ -58,7 +58,10 @@ func printProgress(status string, canUpdateStatus bool) {
 		if w < 3 {
 			status = termstatus.Truncate(status, w)
 		} else {
-			status = termstatus.Truncate(status, w-3) + "..."
+			trunc := termstatus.Truncate(status, w-3)
+			if len(trunc) < len(status) {
+				status = trunc + "..."
+			}
 		}
 	}
 

cmd/restic/secondary_repo.go

@@ -8,49 +8,98 @@ import (
 )
 
 type secondaryRepoOptions struct {
+	password string
+	// from-repo options
 	Repo            string
 	RepositoryFile  string
-	password        string
 	PasswordFile    string
 	PasswordCommand string
 	KeyHint         string
+	// repo2 options
+	LegacyRepo            string
+	LegacyRepositoryFile  string
+	LegacyPasswordFile    string
+	LegacyPasswordCommand string
+	LegacyKeyHint         string
 }
 
 func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repoPrefix string, repoUsage string) {
-	f.StringVarP(&opts.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
-	f.StringVarP(&opts.RepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
-	f.StringVarP(&opts.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
-	f.StringVarP(&opts.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
-	f.StringVarP(&opts.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
+	f.StringVarP(&opts.LegacyRepo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
+	f.StringVarP(&opts.LegacyRepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
+	f.StringVarP(&opts.LegacyPasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
+	f.StringVarP(&opts.LegacyKeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
+	f.StringVarP(&opts.LegacyPasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
+
+	// hide repo2 options
+	_ = f.MarkDeprecated("repo2", "use --repo or --from-repo instead")
+	_ = f.MarkDeprecated("repository-file2", "use --repository-file or --from-repository-file instead")
+	_ = f.MarkHidden("password-file2")
+	_ = f.MarkHidden("key-hint2")
+	_ = f.MarkHidden("password-command2")
+
+	f.StringVarP(&opts.Repo, "from-repo", "", os.Getenv("RESTIC_FROM_REPOSITORY"), "source `repository` "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY)")
+	f.StringVarP(&opts.RepositoryFile, "from-repository-file", "", os.Getenv("RESTIC_FROM_REPOSITORY_FILE"), "`file` from which to read the source repository location "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY_FILE)")
+	f.StringVarP(&opts.PasswordFile, "from-password-file", "", os.Getenv("RESTIC_FROM_PASSWORD_FILE"), "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)")
+	f.StringVarP(&opts.KeyHint, "from-key-hint", "", os.Getenv("RESTIC_FROM_KEY_HINT"), "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)")
+	f.StringVarP(&opts.PasswordCommand, "from-password-command", "", os.Getenv("RESTIC_FROM_PASSWORD_COMMAND"), "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)")
 }
 
-func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, error) {
-	if opts.Repo == "" && opts.RepositoryFile == "" {
-		return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2 or --repository-file2)")
+func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) {
+	if opts.Repo == "" && opts.RepositoryFile == "" && opts.LegacyRepo == "" && opts.LegacyRepositoryFile == "" {
+		return GlobalOptions{}, false, errors.Fatal("Please specify a source repository location (--from-repo or --from-repository-file)")
 	}
 
-	if opts.Repo != "" && opts.RepositoryFile != "" {
-		return GlobalOptions{}, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one")
+	hasFromRepo := opts.Repo != "" || opts.RepositoryFile != "" || opts.PasswordFile != "" ||
+		opts.KeyHint != "" || opts.PasswordCommand != ""
+	hasRepo2 := opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" || opts.LegacyPasswordFile != "" ||
+		opts.LegacyKeyHint != "" || opts.LegacyPasswordCommand != ""
+
+	if hasFromRepo && hasRepo2 {
+		return GlobalOptions{}, false, errors.Fatal("Option groups repo2 and from-repo are mutually exclusive, please specify only one")
 	}
 
 	var err error
 	dstGopts := gopts
-	dstGopts.Repo = opts.Repo
-	dstGopts.RepositoryFile = opts.RepositoryFile
-	dstGopts.PasswordFile = opts.PasswordFile
-	dstGopts.PasswordCommand = opts.PasswordCommand
-	dstGopts.KeyHint = opts.KeyHint
+	var pwdEnv string
+
+	if hasFromRepo {
+		if opts.Repo != "" && opts.RepositoryFile != "" {
+			return GlobalOptions{}, false, errors.Fatal("Options --from-repo and --from-repository-file are mutually exclusive, please specify only one")
+		}
+
+		dstGopts.Repo = opts.Repo
+		dstGopts.RepositoryFile = opts.RepositoryFile
+		dstGopts.PasswordFile = opts.PasswordFile
+		dstGopts.PasswordCommand = opts.PasswordCommand
+		dstGopts.KeyHint = opts.KeyHint
+
+		pwdEnv = "RESTIC_FROM_PASSWORD"
+		repoPrefix = "source"
+	} else {
+		if opts.LegacyRepo != "" && opts.LegacyRepositoryFile != "" {
+			return GlobalOptions{}, false, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one")
+		}
+
+		dstGopts.Repo = opts.LegacyRepo
+		dstGopts.RepositoryFile = opts.LegacyRepositoryFile
+		dstGopts.PasswordFile = opts.LegacyPasswordFile
+		dstGopts.PasswordCommand = opts.LegacyPasswordCommand
+		dstGopts.KeyHint = opts.LegacyKeyHint
+
+		pwdEnv = "RESTIC_PASSWORD2"
+	}
+
 	if opts.password != "" {
 		dstGopts.password = opts.password
 	} else {
-		dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2")
+		dstGopts.password, err = resolvePassword(dstGopts, pwdEnv)
 		if err != nil {
-			return GlobalOptions{}, err
+			return GlobalOptions{}, false, err
 		}
 	}
 	dstGopts.password, err = ReadPassword(dstGopts, "enter password for "+repoPrefix+" repository: ")
 	if err != nil {
-		return GlobalOptions{}, err
+		return GlobalOptions{}, false, err
 	}
-	return dstGopts, nil
+	return dstGopts, hasFromRepo, nil
 }

cmd/restic/secondary_repo_test.go

@@ -8,12 +8,13 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-//TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function
+// TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function
 func TestFillSecondaryGlobalOpts(t *testing.T) {
 	//secondaryRepoTestCase defines a struct for test cases
 	type secondaryRepoTestCase struct {
 		Opts     secondaryRepoOptions
 		DstGOpts GlobalOptions
+		FromRepo bool
 	}
 
 	//validSecondaryRepoTestCases is a list with test cases that must pass
@@ -28,6 +29,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 				Repo:     "backupDst",
 				password: "secretDst",
 			},
+			FromRepo: true,
 		},
 		{
 			// Test if RepositoryFile and PasswordFile are parsed correctly.
@@ -40,6 +42,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 				password:       "secretDst",
 				PasswordFile:   "passwordFileDst",
 			},
+			FromRepo: true,
 		},
 		{
 			// Test if RepositoryFile and PasswordCommand are parsed correctly.
@@ -52,6 +55,42 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 				password:        "secretDst",
 				PasswordCommand: "echo secretDst",
 			},
+			FromRepo: true,
+		},
+		{
+			// Test if LegacyRepo and Password are parsed correctly.
+			Opts: secondaryRepoOptions{
+				LegacyRepo: "backupDst",
+				password:   "secretDst",
+			},
+			DstGOpts: GlobalOptions{
+				Repo:     "backupDst",
+				password: "secretDst",
+			},
+		},
+		{
+			// Test if LegacyRepositoryFile and LegacyPasswordFile are parsed correctly.
+			Opts: secondaryRepoOptions{
+				LegacyRepositoryFile: "backupDst",
+				LegacyPasswordFile:   "passwordFileDst",
+			},
+			DstGOpts: GlobalOptions{
+				RepositoryFile: "backupDst",
+				password:       "secretDst",
+				PasswordFile:   "passwordFileDst",
+			},
+		},
+		{
+			// Test if LegacyRepositoryFile and LegacyPasswordCommand are parsed correctly.
+			Opts: secondaryRepoOptions{
+				LegacyRepositoryFile:  "backupDst",
+				LegacyPasswordCommand: "echo secretDst",
+			},
+			DstGOpts: GlobalOptions{
+				RepositoryFile:  "backupDst",
+				password:        "secretDst",
+				PasswordCommand: "echo secretDst",
+			},
 		},
 	}
 
@@ -96,6 +135,20 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 				Repo: "backupDst",
 			},
 		},
+		{
+			// Test must fail as current and legacy options are mixed
+			Opts: secondaryRepoOptions{
+				Repo:       "backupDst",
+				LegacyRepo: "backupDst",
+			},
+		},
+		{
+			// Test must fail as current and legacy options are mixed
+			Opts: secondaryRepoOptions{
+				Repo:                  "backupDst",
+				LegacyPasswordCommand: "notEmpty",
+			},
+		},
 	}
 
 	//gOpts defines the Global options used in the secondary repository tests
@@ -119,14 +172,15 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 
 	// Test all valid cases
 	for _, testCase := range validSecondaryRepoTestCases {
-		DstGOpts, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
+		DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
 		rtest.OK(t, err)
 		rtest.Equals(t, DstGOpts, testCase.DstGOpts)
+		rtest.Equals(t, isFromRepo, testCase.FromRepo)
 	}
 
 	// Test all invalid cases
 	for _, testCase := range invalidSecondaryRepoTestCases {
-		_, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
+		_, _, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
 		rtest.Assert(t, err != nil, "Expected error, but function did not return an error")
 	}
 }

doc.go

@@ -1,6 +1,6 @@
 // Package restic gives a (very brief) introduction to the structure of source code.
 //
-// Overview
+// # Overview
 //
 // The packages are structured so that cmd/ contains the main package for the
 // restic binary, and internal/ contains almost all code in library form. We've

doc/020_installation.rst

@@ -274,7 +274,7 @@ From Source
 ***********
 
 restic is written in the Go programming language and you need at least
-Go version 1.14. Building restic may also work with older versions of Go,
+Go version 1.15. Building restic may also work with older versions of Go,
 but that's not supported. See the `Getting
 started <https://golang.org/doc/install>`__ guide of the Go project for
 instructions how to install Go.
@@ -339,6 +339,13 @@ Example for using sudo to write a bash completion script directly to the system-
     $ sudo ./restic generate --bash-completion /etc/bash_completion.d/restic
     writing bash completion file to /etc/bash_completion.d/restic
 
+Example for using sudo to write a zsh completion script directly to the system-wide location:
+
+.. code-block:: console
+
+    $ sudo ./restic generate --zsh-completion /usr/local/share/zsh/site-functions/_restic
+    writing zsh completion file to /usr/local/share/zsh/site-functions/_restic
+
 .. note:: The path for the ``--bash-completion`` option may vary depending on
    the operating system used, e.g. ``/usr/share/bash-completion/completions/restic``
    in Debian and derivatives. Please look up the correct path in the appropriate

doc/030_preparing_a_new_repo.rst

@@ -14,18 +14,26 @@
 Preparing a new repository
 ##########################
 
-The place where your backups will be saved is called a "repository".
+The place where your backups will be saved is called a "repository". This is
+simply a directory containing a set of subdirectories and files created by
+restic to store your backups, some corresponding metadata and encryption keys.
+
+To access the repository, a password (also called a key) must be specified. A
+repository can hold multiple keys that can all be used to access the repository.
+
 This chapter explains how to create ("init") such a repository. The repository
 can be stored locally, or on some remote server or service. We'll first cover
 using a local repository; the remaining sections of this chapter cover all the
 other options. You can skip to the next chapter once you've read the relevant
 section here.
 
-For automated backups, restic accepts the repository location in the
+For automated backups, restic supports specifying the repository location in the
 environment variable ``RESTIC_REPOSITORY``. Restic can also read the repository
 location from a file specified via the ``--repository-file`` option or the
-environment variable ``RESTIC_REPOSITORY_FILE``. For the password, several
-options exist:
+environment variable ``RESTIC_REPOSITORY_FILE``.
+
+For automating the supply of the repository password to restic, several options
+exist:
 
  * Setting the environment variable ``RESTIC_PASSWORD``
 
@@ -35,6 +43,26 @@ options exist:
  * Configuring a program to be called when the password is needed via the
    option ``--password-command`` or the environment variable
    ``RESTIC_PASSWORD_COMMAND``
+   
+The ``init`` command has an option called ``--repository-version`` which can
+be used to explicitly set the version of the new repository. By default, the
+current stable version is used (see table below). The alias ``latest`` will
+always resolve to the latest repository version. Have a look at the `design
+documentation <https://github.com/restic/restic/blob/master/doc/design.rst>`__
+for more details.
+
+The below table shows which restic version is required to use a certain
+repository version, as well as notable features introduced in the various
+versions.
+
++--------------------+-------------------------+---------------------+------------------+
+| Repository version | Required restic version | Major new features  | Comment          |
++====================+=========================+=====================+==================+
+| ``1``              | Any                     |                     | Current default  |
++--------------------+-------------------------+---------------------+------------------+
+| ``2``              | 0.14.0 or newer         | Compression support |                  |
++--------------------+-------------------------+---------------------+------------------+
+
 
 Local
 *****
@@ -67,9 +95,9 @@ SFTP
 ****
 
 In order to backup data via SFTP, you must first set up a server with
-SSH and let it know your public key. Passwordless login is really
-important since restic fails to connect to the repository if the server
-prompts for credentials.
+SSH and let it know your public key. Passwordless login is important
+since automatic backups are not possible if the server prompts for
+credentials.
 
 Once the server is configured, the setup of the SFTP repository can
 simply be achieved by changing the URL scheme in the ``init`` command:
@@ -148,7 +176,7 @@ SFTP connection, you can specify the command to be run with the option
 .. note:: Please be aware that sftp servers close connections when no data is
           received by the client. This can happen when restic is processing huge
           amounts of unchanged data. To avoid this issue add the following lines 
-          to the client’s .ssh/config file:
+          to the client's .ssh/config file:
 
 ::
 
@@ -167,7 +195,7 @@ scheme like this:
 
 .. code-block:: console
 
-    $ restic -r rest:http://host:8000/
+    $ restic -r rest:http://host:8000/ init
 
 Depending on your REST server setup, you can use HTTPS protocol,
 password protection, multiple repositories or any combination of
@@ -176,9 +204,9 @@ are some more examples:
 
 .. code-block:: console
 
-    $ restic -r rest:https://host:8000/
-    $ restic -r rest:https://user:pass@host:8000/
-    $ restic -r rest:https://user:pass@host:8000/my_backup_repo/
+    $ restic -r rest:https://host:8000/ init
+    $ restic -r rest:https://user:pass@host:8000/ init
+    $ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init
 
 If you use TLS, restic will use the system's CA certificates to verify the
 server certificate. When the verification fails, restic refuses to proceed and
@@ -226,6 +254,9 @@ parameter like ``-o s3.region="us-east-1"``. If the region is not specified,
 the default region is used. Afterwards, the S3 server (at least for AWS,
 ``s3.amazonaws.com``) will redirect restic to the correct endpoint.
 
+When using temporary credentials make sure to include the session token via
+then environment variable ``AWS_SESSION_TOKEN``.
+
 Until version 0.8.0, restic used a default prefix of ``restic``, so the files
 in the bucket were placed in a directory named ``restic``. If you want to
 access a repository created with an older version of restic, specify the path
@@ -482,6 +513,13 @@ account name and key as follows:
     $ export AZURE_ACCOUNT_NAME=<ACCOUNT_NAME>
     $ export AZURE_ACCOUNT_KEY=<SECRET_KEY>
 
+or
+
+.. code-block:: console
+
+    $ export AZURE_ACCOUNT_NAME=<ACCOUNT_NAME>
+    $ export AZURE_ACCOUNT_SAS=<SAS_TOKEN>
+
 Afterwards you can initialize a repository in a container called ``foo`` in the
 root path like this:
 
@@ -501,6 +539,10 @@ established.
 Google Cloud Storage
 ********************
 
+.. note:: Google Cloud Storage is not the same service as Google Drive - to use
+          the latter, please see :ref:`other-services` for instructions on using
+          the rclone backend.
+
 Restic supports Google Cloud Storage as a backend and connects via a `service account`_.
 
 For normal restic operation, the service account must have the
@@ -547,16 +589,18 @@ repository in the bucket ``foo`` at the root path:
     enter password for new repository:
     enter password again:
 
-    created restic repository bde47d6254 at gs:foo2/
+    created restic repository bde47d6254 at gs:foo/
     [...]
 
 The number of concurrent connections to the GCS service can be set with the
 ``-o gs.connections=10`` switch. By default, at most five parallel connections are
 established.
 
-.. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts
-.. _create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key
-.. _default authentication material: https://developers.google.com/identity/protocols/application-default-credentials
+.. _service account: https://cloud.google.com/iam/docs/service-accounts
+.. _create a service account key: https://cloud.google.com/iam/docs/creating-managing-service-account-keys#iam-service-account-keys-create-console
+.. _default authentication material: https://cloud.google.com/docs/authentication/production
+
+.. _other-services:
 
 Other Services via rclone
 *************************
@@ -566,7 +610,7 @@ store data there. First, you need to install and `configure`_ rclone.  The
 general backend specification format is ``rclone:<remote>:<path>``, the
 ``<remote>:<path>`` component will be directly passed to rclone. When you
 configure a remote named ``foo``, you can then call restic as follows to
-initiate a new repository in the path ``bar`` in the repo:
+initiate a new repository in the path ``bar`` in the remote ``foo``:
 
 .. code-block:: console
 
@@ -693,3 +737,55 @@ On MSYS2, you can install ``winpty`` as follows:
     $ pacman -S winpty
     $ winpty restic -r /srv/restic-repo init
 
+
+Group accessible repositories
+*****************************
+
+Since restic version 0.14 local and SFTP repositories can be made
+accessible to members of a system group. To control this we have to change
+the group permissions of the top-level ``config`` file and restic will use
+this as a hint to determine what permissions to apply to newly created
+files. By default ``restic init`` sets repositories up to be group
+inaccessible.
+
+In order to give group members read-only access we simply add the read
+permission bit to all repository files with ``chmod``:
+
+.. code-block:: console
+
+    $ chmod -R g+r /srv/restic-repo
+
+This serves two purposes: 1) it sets the read permission bit on the
+repository config file triggering restic's logic to create new files as
+group accessible and 2) it actually allows the group read access to the
+files.
+
+.. note:: By default files on Unix systems are created with a user's
+          primary group as defined by the gid (group id) field in
+          ``/etc/passwd``. See `passwd(5)
+          <https://manpages.debian.org/latest/passwd/passwd.5.en.html>`_.
+
+For read-write access things are a bit more complicated. When users other
+than the repository creator add new files in the repository they will be
+group-owned by this user's primary group by default, not that of the
+original repository owner, meaning the original creator wouldn't have
+access to these files. That's hardly what you'd want.
+
+To make this work we can employ the help of the ``setgid`` permission bit
+available on Linux and most other Unix systems. This permission bit makes
+newly created directories inherit both the group owner (gid) and setgid bit
+from the parent directory. Setting this bit requires root but since it
+propagates down to any new directories we only have to do this priviledged
+setup once:
+
+.. code-block:: console
+
+    # find /srv/restic-repo -type d -exec chmod g+s '{}' \;
+    $ chmod -R g+rw /srv/restic-repo
+
+This sets the ``setgid`` bit on all existing directories in the repository
+and then grants read/write permissions for group access.
+
+.. note:: To manage who has access to the repository you can use
+          ``usermod`` on Linux systems, to change which group controls
+          repository access ``chgrp -R`` is your friend.

doc/040_backup.rst

@@ -191,7 +191,7 @@ Dry Runs
 ********
 
 You can perform a backup in dry run mode to see what would happen without
-modifying the repo.
+modifying the repository.
 
 -  ``--dry-run``/``-n`` Report what would be done, without writing to the repository
 
@@ -202,7 +202,7 @@ Combined with ``--verbose``, you can see a list of changes:
     $ restic -r /srv/restic-repo backup ~/work --dry-run -vv | grep "added"
     modified  /plan.txt, saved in 0.000s (9.110 KiB added)
     modified  /archive.tar.gz, saved in 0.140s (25.542 MiB added)
-    Would be added to the repo: 25.551 MiB
+    Would be added to the repository: 25.551 MiB
 
 Excluding Files
 ***************
@@ -212,7 +212,7 @@ the exclude options are:
 
 -  ``--exclude`` Specified one or more times to exclude one or more items
 -  ``--iexclude`` Same as ``--exclude`` but ignores the case of paths
--  ``--exclude-caches`` Specified once to exclude folders containing a special file
+-  ``--exclude-caches`` Specified once to exclude folders containing `this special file <https://bford.info/cachedir/>`__
 -  ``--exclude-file`` Specified one or more times to exclude items listed in a given file
 -  ``--iexclude-file`` Same as ``exclude-file`` but ignores cases like in ``--iexclude``
 -  ``--exclude-if-present foo`` Specified one or more times to exclude a folder's content if it contains a file called ``foo`` (optionally having a given header, no wildcards for the file name supported)
@@ -346,12 +346,12 @@ option:
 
     $ restic -r /srv/restic-repo backup ~/work --exclude-larger-than 1M
 
-This excludes files in ``~/work`` which are larger than 1 MB from the backup.
+This excludes files in ``~/work`` which are larger than 1 MiB from the backup.
 
 The default unit for the size value is bytes, so e.g. ``--exclude-larger-than 2048``
-would exclude files larger than 2048 bytes (2 kilobytes). To specify other units,
-suffix the size value with one of ``k``/``K`` for kilobytes, ``m``/``M`` for megabytes,
-``g``/``G`` for gigabytes and ``t``/``T`` for terabytes (e.g. ``1k``, ``10K``, ``20m``,
+would exclude files larger than 2048 bytes (2 KiB). To specify other units,
+suffix the size value with one of ``k``/``K`` for KiB (1024 bytes), ``m``/``M`` for MiB (1024^2 bytes),
+``g``/``G`` for GiB (1024^3 bytes) and ``t``/``T`` for TiB (1024^4 bytes), e.g. ``1k``, ``10K``, ``20m``,
 ``20M``,  ``30g``, ``30G``, ``2t`` or ``2T``).
 
 Including Files
@@ -552,12 +552,15 @@ environment variables. The following lists these environment variables:
     RESTIC_PASSWORD_COMMAND             Command printing the password for the repository to stdout
     RESTIC_KEY_HINT                     ID of key to try decrypting first, before other keys
     RESTIC_CACHE_DIR                    Location of the cache directory
+    RESTIC_COMPRESSION                  Compression mode (only available for repository format version 2)
     RESTIC_PROGRESS_FPS                 Frames per second by which the progress bar is updated
+    RESTIC_PACK_SIZE                    Target size for pack files
 
     TMPDIR                              Location for temporary files
 
     AWS_ACCESS_KEY_ID                   Amazon S3 access key ID
     AWS_SECRET_ACCESS_KEY               Amazon S3 secret access key
+    AWS_SESSION_TOKEN                   Amazon S3 temporary session token
     AWS_DEFAULT_REGION                  Amazon S3 default region
     AWS_PROFILE                         Amazon credentials profile (alternative to specifying key and region)
     AWS_SHARED_CREDENTIALS_FILE         Location of the AWS CLI shared credentials file (default: ~/.aws/credentials)
@@ -593,6 +596,7 @@ environment variables. The following lists these environment variables:
 
     AZURE_ACCOUNT_NAME                  Account name for Azure
     AZURE_ACCOUNT_KEY                   Account key for Azure
+    AZURE_ACCOUNT_SAS                   Shared access signatures (SAS) for Azure
 
     GOOGLE_PROJECT_ID                   Project ID for Google Cloud Storage
     GOOGLE_APPLICATION_CREDENTIALS      Application Credentials for Google Cloud Storage (e.g. $HOME/.config/gs-secret-restic-key.json)

doc/045_working_with_repos.rst

@@ -90,7 +90,7 @@ example from a local to a remote repository, you can use the ``copy`` command:
 
 .. code-block:: console
 
-    $ restic -r /srv/restic-repo copy --repo2 /srv/restic-repo-copy
+    $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo
     repository d6504c63 opened successfully, password is correct
     repository 3dd0878c opened successfully, password is correct
 
@@ -117,17 +117,17 @@ be skipped by later copy runs.
     both the source and destination repository, *may occupy up to twice their
     space* in the destination repository. See below for how to avoid this.
 
-The destination repository is specified with ``--repo2`` or can be read 
-from a file specified via ``--repository-file2``. Both of these options
-can also set as environment variables ``$RESTIC_REPOSITORY2`` or
-``$RESTIC_REPOSITORY_FILE2`` respectively. For the destination repository 
-the password can be read from a file ``--password-file2`` or from a command 
-``--password-command2``.
-Alternatively the environment variables ``$RESTIC_PASSWORD_COMMAND2`` and
-``$RESTIC_PASSWORD_FILE2`` can be used. It is also possible to directly
-pass the password via ``$RESTIC_PASSWORD2``. The key which should be used
-for decryption can be selected by passing its ID via the flag ``--key-hint2``
-or the environment variable ``$RESTIC_KEY_HINT2``.
+The source repository is specified with ``--from-repo`` or can be read
+from a file specified via ``--from-repository-file``. Both of these options
+can also be set as environment variables ``$RESTIC_FROM_REPOSITORY`` or
+``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the destination repository
+the password can be read from a file ``--from-password-file`` or from a command
+``--from-password-command``.
+Alternatively the environment variables ``$RESTIC_FROM_PASSWORD_COMMAND`` and
+``$RESTIC_FROM_PASSWORD_FILE`` can be used. It is also possible to directly
+pass the password via ``$RESTIC_FROM_PASSWORD``. The key which should be used
+for decryption can be selected by passing its ID via the flag ``--from-key-hint``
+or the environment variable ``$RESTIC_FROM_KEY_HINT``.
 
 .. note:: In case the source and destination repository use the same backend,
     the configuration options and environment variables used to configure the
@@ -298,3 +298,29 @@ a file size value the following command may be used:
 
     $ restic -r /srv/restic-repo check --read-data-subset=50M
     $ restic -r /srv/restic-repo check --read-data-subset=10G
+
+
+Upgrading the repository format version
+=======================================
+
+Repositories created using earlier restic versions use an older repository
+format version and have to be upgraded to allow using all new features.
+Upgrading must be done explicitly as a newer repository version increases the
+minimum restic version required to access the repository. For example the
+repository format version 2 is only readable using restic 0.14.0 or newer.
+
+Upgrading to repository version 2 is a two step process: first run
+``migrate upgrade_repo_v2`` which will check the repository integrity and
+then upgrade the repository version. Repository problems must be corrected
+before the migration will be possible. After the migration is complete, run
+``prune`` to compress the repository metadata. To limit the amount of data
+rewritten in at once, you can use the ``prune --max-repack-size size``
+parameter, see :ref:`customize-pruning` for more details.
+
+File contents stored in the repository will not be rewritten, data from new
+backups will be compressed. Over time more and more of the repository will
+be compressed. To speed up this process and compress all not yet compressed
+data, you can run ``prune --repack-uncompressed``. When you plan to create
+your backups with maximum compression, you should also add the
+``--compression max`` flag to the prune command. For already backed up data,
+the compression level cannot be changed later on.

doc/047_tuning_backup_parameters.rst

@@ -0,0 +1,80 @@
+..
+  Normally, there are no heading levels assigned to certain characters as the structure is
+  determined from the succession of headings. However, this convention is used in Python’s
+  Style Guide for documenting which you may follow:
+  # with overline, for parts
+  * for chapters
+  = for sections
+  - for subsections
+  ^ for subsubsections
+  " for paragraphs
+
+########################
+Tuning Backup Parameters
+########################
+
+Restic offers a few parameters that allow tuning the backup. The default values should
+work well in general although specific use cases can benefit from different non-default
+values. As the restic commands evolve over time, the optimal value for each parameter
+can also change across restic versions.
+
+
+Backend Connections
+===================
+
+Restic uses a global limit for the number of concurrent connections to a backend.
+This limit can be configured using ``-o <backend-name>.connections=5``, for example for
+the REST backend the parameter would be ``-o rest.connections=5``. By default restic uses
+``5`` connections for each backend, except for the local backend which uses a limit of ``2``.
+The defaults should work well in most cases. For high-latency backends it can be beneficial
+to increase the number of connections. Please be aware that this increases the resource
+consumption of restic and that a too high connection count *will degrade performance*.
+
+
+CPU Usage
+=========
+
+By default, restic uses all available CPU cores. You can set the environment variable
+`GOMAXPROCS` to limit the number of used CPU cores. For example to use a single CPU core,
+use `GOMAXPROCS=1`. Limiting the number of usable CPU cores, can slightly reduce the memory
+usage of restic.
+
+
+Compression
+===========
+
+For a repository using at least repository format version 2, you can configure how data
+is compressed with the option ``--compression``. It can be set to ``auto`` (the default,
+which will compress very fast), ``max`` (which will trade backup speed and CPU usage for
+slightly better compression), or ``off`` (which disables compression). Each setting is
+only applied for the single run of restic. The option can also be set via the environment
+variable ``RESTIC_COMPRESSION``.
+
+
+Pack Size
+=========
+
+In certain instances, such as very large repositories (in the TiB range) or very fast
+upload connections, it is desirable to use larger pack sizes to reduce the number of
+files in the repository and improve upload performance.  Notable examples are OpenStack
+Swift and some Google Drive Team accounts, where there are hard limits on the total
+number of files.  Larger pack sizes can also improve the backup speed for a repository
+stored on a local HDD.  This can be achieved by either using the ``--pack-size`` option
+or defining the ``$RESTIC_PACK_SIZE`` environment variable.  Restic currently defaults
+to a 16 MiB pack size.
+
+The side effect of increasing the pack size is requiring more disk space for temporary pack
+files created before uploading.  The space must be available in the system default temp
+directory, unless overwritten by setting the ``$TMPDIR`` environment variable.  In addition,
+depending on the backend the memory usage can also increase by a similar amount. Restic
+requires temporary space according to the pack size, multiplied by the number
+of backend connections plus one. For example, if the backend uses 5 connections (the default
+for most backends), with a target pack size of 64 MiB, you'll need a *minimum* of 384 MiB
+of space in the temp directory. A bit of tuning may be required to strike a balance between
+resource usage at the backup client and the number of pack files in the repository.
+
+Note that larger pack files increase the chance that the temporary pack files are written
+to disk. An operating system usually caches file write operations in memory and writes
+them to disk after a short delay. As larger pack files take longer to upload, this
+increases the chance of these files being written to disk. This can increase disk wear
+for SSDs.

doc/050_restore.rst

@@ -96,7 +96,7 @@ the data directly. This can be achieved by using the `dump` command, like this:
 
 If you have saved multiple different things into the same repo, the ``latest``
 snapshot may not be the right one. For example, consider the following
-snapshots in a repo:
+snapshots in a repository:
 
 .. code-block:: console
 

doc/060_forget.rst

@@ -212,12 +212,13 @@ The ``forget`` command accepts the following policy options:
 
 .. note:: Specifying ``--keep-tag ''`` will match untagged snapshots only.
 
-When ``forget`` is run with a policy, restic loads the list of all snapshots,
-then groups these by host name and list of directories. The grouping options can
-be set with ``--group-by``, to e.g. group snapshots by only paths and tags use
-``--group-by paths,tags``. The policy is then applied to each group of snapshots
-separately. This is a safety feature to prevent accidental removal of unrelated
-backup sets.
+When ``forget`` is run with a policy, restic first loads the list of all snapshots
+and groups them by their host name and paths. The grouping options can be set with
+``--group-by``, e.g. using ``--group-by paths,tags`` to instead group snapshots by
+paths and tags. The policy is then applied to each group of snapshots individually.
+This is a safety feature to prevent accidental removal of unrelated backup sets. To
+disable grouping and apply the policy to all snapshots regardless of their host,
+paths and tags, use ``--group-by ''`` (that is, an empty value to ``--group-by``).
 
 Additionally, you can restrict the policy to only process snapshots which have a
 particular hostname with the ``--host`` parameter, or tags with the ``--tag``
@@ -387,6 +388,8 @@ the specified duration: if ``forget --keep-within 7d`` is run 8 days after the
 last good snapshot, then the attacker can still use that opportunity to remove
 all legitimate snapshots.
 
+.. _customize-pruning:
+
 Customize pruning
 *****************
 
@@ -415,9 +418,9 @@ The ``prune`` command accepts the following options:
 
     * As an absolute size (e.g. ``200M``). If you want to minimize the space
       used by your repository, pass ``0`` to this option.
-    * As a size relative to the total repo size (e.g. ``10%``). This means that
-      after prune, at most ``10%`` of the total data stored in the repo may be
-      unused data. If the repo after prune has as size of 500MB, then at most
+    * As a size relative to the total repository size (e.g. ``10%``). This means that
+      after prune, at most ``10%`` of the total data stored in the repository may be
+      unused data. If the repository after prune has a size of 500MB, then at most
       50MB may be unused.
     * If the string ``unlimited`` is passed, there is no limit for partly
       unused files. This means that as long as some data is still used within
@@ -443,3 +446,31 @@ The ``prune`` command accepts the following options:
 -  ``--dry-run`` only show what ``prune`` would do.
 
 -  ``--verbose`` increased verbosity shows additional statistics for ``prune``.
+
+
+Recovering from "no free space" errors
+**************************************
+
+In some cases when a repository has grown large enough to fill up all disk space or the
+allocated quota, then ``prune`` might fail to free space. ``prune`` works in such a way
+that a repository remains usable no matter at which point the command is interrupted.
+However, this also means that ``prune`` requires some scratch space to work.
+
+In most cases it is sufficient to instruct ``prune`` to use as little scratch space as
+possible by running it as ``prune --max-repack-size 0``. Note that for restic versions
+before 0.13.0 ``prune --max-repack-size 1`` must be used. Obviously, this can only work
+if several snapshots have been removed using ``forget`` before. This then allows the
+``prune`` command to actually remove data from the repository. If the command succeeds,
+but there is still little free space, then remove a few more snapshots and run ``prune`` again.
+
+If ``prune`` fails to complete, then ``prune --unsafe-recover-no-free-space SOME-ID``
+is available as a method of last resort. It allows prune to work with little to no free
+space. However, a **failed** ``prune`` run can cause the repository to become
+**temporarily unusable**. Therefore, make sure that you have a stable connection to the
+repository storage, before running this command. In case the command fails, it may become
+necessary to manually remove all files from the `index/` folder of the repository and
+run `rebuild-index` afterwards.
+
+To prevent accidental usages of the ``--unsafe-recover-no-free-space`` option it is
+necessary to first run ``prune --unsafe-recover-no-free-space SOME-ID`` and then replace
+``SOME-ID`` with the requested ID.

doc/080_examples.rst

@@ -202,11 +202,12 @@ configuration of restic will be placed into environment variables. This will
 include sensitive information, such as your AWS secret and repository password.
 Therefore, make sure the next commands **do not** end up in your shell's
 history file. Adjust the contents of the environment variables to fit your
-bucket's name and your user's API credentials.
+bucket's name, region, and your user's API credentials.
 
 .. code-block:: console
 
    $ unset HISTFILE
+   $ export AWS_DEFAULT_REGION="eu-west-1"
    $ export RESTIC_REPOSITORY="s3:https://s3.amazonaws.com/restic-demo"
    $ export AWS_ACCESS_KEY_ID="AKIAJAJSLTZCAZ4SRI5Q"
    $ export AWS_SECRET_ACCESS_KEY="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"

doc/090_participating.rst

@@ -14,18 +14,12 @@
 Participating
 #############
 
-*********
-Debugging
-*********
+**********
+Debug Logs
+**********
 
-The program can be built with debug support like this:
-
-.. code-block:: console
-
-    $ go run build.go -tags debug
-
-Afterwards, extensive debug messages are written to the file in
-environment variable ``DEBUG_LOG``, e.g.:
+Set the environment variable ``DEBUG_LOG`` to let restic write extensive debug
+messages to the specified filed, e.g.:
 
 .. code-block:: console
 
@@ -66,6 +60,21 @@ statements originating in functions that match the pattern ``*unlock*``
     $ DEBUG_FUNCS=*unlock* restic check
 
 
+*********
+Debugging
+*********
+
+The program can be built with debug support like this:
+
+.. code-block:: console
+
+    $ go run build.go -tags debug
+
+This will make the ``restic debug <subcommand>`` available which can be used to
+inspect internal data structures. In addition, this enables profiling support
+which can help with investigation performance and memory usage issues.
+
+
 ************
 Contributing
 ************

doc/bash-completion.sh

@@ -2,7 +2,7 @@
 
 __restic_debug()
 {
-    if [[ -n ${BASH_COMP_DEBUG_FILE} ]]; then
+    if [[ -n ${BASH_COMP_DEBUG_FILE:-} ]]; then
         echo "$*" >> "${BASH_COMP_DEBUG_FILE}"
     fi
 }
@@ -51,7 +51,8 @@ __restic_handle_go_custom_completion()
     # Prepare the command to request completions for the program.
     # Calling ${words[0]} instead of directly restic allows to handle aliases
     args=("${words[@]:1}")
-    requestComp="${words[0]} __completeNoDesc ${args[*]}"
+    # Disable ActiveHelp which is not supported for bash completion v1
+    requestComp="RESTIC_ACTIVE_HELP=0 ${words[0]} __completeNoDesc ${args[*]}"
 
     lastParam=${words[$((${#words[@]}-1))]}
     lastChar=${lastParam:$((${#lastParam}-1)):1}
@@ -77,7 +78,7 @@ __restic_handle_go_custom_completion()
         directive=0
     fi
     __restic_debug "${FUNCNAME[0]}: the completion directive is: ${directive}"
-    __restic_debug "${FUNCNAME[0]}: the completions are: ${out[*]}"
+    __restic_debug "${FUNCNAME[0]}: the completions are: ${out}"
 
     if [ $((directive & shellCompDirectiveError)) -ne 0 ]; then
         # Error code.  No completion.
@@ -103,7 +104,7 @@ __restic_handle_go_custom_completion()
         local fullFilter filter filteringCmd
         # Do not use quotes around the $out variable or else newline
         # characters will be kept.
-        for filter in ${out[*]}; do
+        for filter in ${out}; do
             fullFilter+="$filter|"
         done
 
@@ -112,9 +113,9 @@ __restic_handle_go_custom_completion()
         $filteringCmd
     elif [ $((directive & shellCompDirectiveFilterDirs)) -ne 0 ]; then
         # File completion for directories only
-        local subDir
+        local subdir
         # Use printf to strip any trailing newline
-        subdir=$(printf "%s" "${out[0]}")
+        subdir=$(printf "%s" "${out}")
         if [ -n "$subdir" ]; then
             __restic_debug "Listing directories in $subdir"
             __restic_handle_subdirs_in_dir_flag "$subdir"
@@ -125,7 +126,7 @@ __restic_handle_go_custom_completion()
     else
         while IFS='' read -r comp; do
             COMPREPLY+=("$comp")
-        done < <(compgen -W "${out[*]}" -- "$cur")
+        done < <(compgen -W "${out}" -- "$cur")
     fi
 }
 
@@ -165,13 +166,19 @@ __restic_handle_reply()
                     PREFIX=""
                     cur="${cur#*=}"
                     ${flags_completion[${index}]}
-                    if [ -n "${ZSH_VERSION}" ]; then
+                    if [ -n "${ZSH_VERSION:-}" ]; then
                         # zsh completion needs --flag= prefix
                         eval "COMPREPLY=( \"\${COMPREPLY[@]/#/${flag}=}\" )"
                     fi
                 fi
             fi
-            return 0;
+
+            if [[ -z "${flag_parsing_disabled}" ]]; then
+                # If flag parsing is enabled, we have completed the flags and can return.
+                # If flag parsing is disabled, we may not know all (or any) of the flags, so we fallthrough
+                # to possibly call handle_go_custom_completion.
+                return 0;
+            fi
             ;;
     esac
 
@@ -210,13 +217,13 @@ __restic_handle_reply()
     fi
 
     if [[ ${#COMPREPLY[@]} -eq 0 ]]; then
-		if declare -F __restic_custom_func >/dev/null; then
-			# try command name qualified custom func
-			__restic_custom_func
-		else
-			# otherwise fall back to unqualified for compatibility
-			declare -F __custom_func >/dev/null && __custom_func
-		fi
+        if declare -F __restic_custom_func >/dev/null; then
+            # try command name qualified custom func
+            __restic_custom_func
+        else
+            # otherwise fall back to unqualified for compatibility
+            declare -F __custom_func >/dev/null && __custom_func
+        fi
     fi
 
     # available in bash-completion >= 2, not always present on macOS
@@ -250,7 +257,7 @@ __restic_handle_flag()
 
     # if a command required a flag, and we found it, unset must_have_one_flag()
     local flagname=${words[c]}
-    local flagvalue
+    local flagvalue=""
     # if the word contained an =
     if [[ ${words[c]} == *"="* ]]; then
         flagvalue=${flagname#*=} # take in as flagvalue after the =
@@ -269,7 +276,7 @@ __restic_handle_flag()
 
     # keep flag value with flagname as flaghash
     # flaghash variable is an associative array which is only supported in bash > 3.
-    if [[ -z "${BASH_VERSION}" || "${BASH_VERSINFO[0]}" -gt 3 ]]; then
+    if [[ -z "${BASH_VERSION:-}" || "${BASH_VERSINFO[0]:-}" -gt 3 ]]; then
         if [ -n "${flagvalue}" ] ; then
             flaghash[${flagname}]=${flagvalue}
         elif [ -n "${words[ $((c+1)) ]}" ] ; then
@@ -281,7 +288,7 @@ __restic_handle_flag()
 
     # skip the argument to a two word flag
     if [[ ${words[c]} != *"="* ]] && __restic_contains_word "${words[c]}" "${two_word_flags[@]}"; then
-			  __restic_debug "${FUNCNAME[0]}: found a flag ${words[c]}, skip the next argument"
+        __restic_debug "${FUNCNAME[0]}: found a flag ${words[c]}, skip the next argument"
         c=$((c+1))
         # if we are looking for a flags value, don't show commands
         if [[ $c -eq $cword ]]; then
@@ -341,7 +348,7 @@ __restic_handle_word()
         __restic_handle_command
     elif __restic_contains_word "${words[c]}" "${command_aliases[@]}"; then
         # aliashash variable is an associative array which is only supported in bash > 3.
-        if [[ -z "${BASH_VERSION}" || "${BASH_VERSINFO[0]}" -gt 3 ]]; then
+        if [[ -z "${BASH_VERSION:-}" || "${BASH_VERSINFO[0]:-}" -gt 3 ]]; then
             words[c]=${aliashash[${words[c]}]}
             __restic_handle_command
         else
@@ -458,6 +465,8 @@ _restic_backup()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -471,6 +480,8 @@ _restic_backup()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -524,6 +535,8 @@ _restic_cache()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -537,6 +550,8 @@ _restic_cache()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -582,6 +597,8 @@ _restic_cat()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -595,6 +612,8 @@ _restic_cat()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -631,8 +650,6 @@ _restic_check()
     flags_with_completion=()
     flags_completion=()
 
-    flags+=("--check-unused")
-    local_nonpersistent_flags+=("--check-unused")
     flags+=("--help")
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
@@ -650,6 +667,8 @@ _restic_check()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -663,6 +682,8 @@ _restic_check()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -699,6 +720,26 @@ _restic_copy()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--from-key-hint=")
+    two_word_flags+=("--from-key-hint")
+    local_nonpersistent_flags+=("--from-key-hint")
+    local_nonpersistent_flags+=("--from-key-hint=")
+    flags+=("--from-password-command=")
+    two_word_flags+=("--from-password-command")
+    local_nonpersistent_flags+=("--from-password-command")
+    local_nonpersistent_flags+=("--from-password-command=")
+    flags+=("--from-password-file=")
+    two_word_flags+=("--from-password-file")
+    local_nonpersistent_flags+=("--from-password-file")
+    local_nonpersistent_flags+=("--from-password-file=")
+    flags+=("--from-repo=")
+    two_word_flags+=("--from-repo")
+    local_nonpersistent_flags+=("--from-repo")
+    local_nonpersistent_flags+=("--from-repo=")
+    flags+=("--from-repository-file=")
+    two_word_flags+=("--from-repository-file")
+    local_nonpersistent_flags+=("--from-repository-file")
+    local_nonpersistent_flags+=("--from-repository-file=")
     flags+=("--help")
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
@@ -709,30 +750,10 @@ _restic_copy()
     local_nonpersistent_flags+=("--host")
     local_nonpersistent_flags+=("--host=")
     local_nonpersistent_flags+=("-H")
-    flags+=("--key-hint2=")
-    two_word_flags+=("--key-hint2")
-    local_nonpersistent_flags+=("--key-hint2")
-    local_nonpersistent_flags+=("--key-hint2=")
-    flags+=("--password-command2=")
-    two_word_flags+=("--password-command2")
-    local_nonpersistent_flags+=("--password-command2")
-    local_nonpersistent_flags+=("--password-command2=")
-    flags+=("--password-file2=")
-    two_word_flags+=("--password-file2")
-    local_nonpersistent_flags+=("--password-file2")
-    local_nonpersistent_flags+=("--password-file2=")
     flags+=("--path=")
     two_word_flags+=("--path")
     local_nonpersistent_flags+=("--path")
     local_nonpersistent_flags+=("--path=")
-    flags+=("--repo2=")
-    two_word_flags+=("--repo2")
-    local_nonpersistent_flags+=("--repo2")
-    local_nonpersistent_flags+=("--repo2=")
-    flags+=("--repository-file2=")
-    two_word_flags+=("--repository-file2")
-    local_nonpersistent_flags+=("--repository-file2")
-    local_nonpersistent_flags+=("--repository-file2=")
     flags+=("--tag=")
     two_word_flags+=("--tag")
     local_nonpersistent_flags+=("--tag")
@@ -742,6 +763,8 @@ _restic_copy()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -755,6 +778,8 @@ _restic_copy()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -802,6 +827,8 @@ _restic_diff()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -815,6 +842,8 @@ _restic_diff()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -880,6 +909,8 @@ _restic_dump()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -893,6 +924,8 @@ _restic_dump()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -986,6 +1019,8 @@ _restic_find()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -999,6 +1034,8 @@ _restic_find()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1137,6 +1174,10 @@ _restic_forget()
     local_nonpersistent_flags+=("--max-repack-size=")
     flags+=("--repack-cacheable-only")
     local_nonpersistent_flags+=("--repack-cacheable-only")
+    flags+=("--repack-small")
+    local_nonpersistent_flags+=("--repack-small")
+    flags+=("--repack-uncompressed")
+    local_nonpersistent_flags+=("--repack-uncompressed")
     flags+=("--help")
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
@@ -1146,6 +1187,8 @@ _restic_forget()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1159,6 +1202,8 @@ _restic_forget()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1220,6 +1265,8 @@ _restic_generate()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1233,6 +1280,8 @@ _restic_generate()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1274,6 +1323,8 @@ _restic_help()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1287,6 +1338,8 @@ _restic_help()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1326,35 +1379,41 @@ _restic_init()
 
     flags+=("--copy-chunker-params")
     local_nonpersistent_flags+=("--copy-chunker-params")
+    flags+=("--from-key-hint=")
+    two_word_flags+=("--from-key-hint")
+    local_nonpersistent_flags+=("--from-key-hint")
+    local_nonpersistent_flags+=("--from-key-hint=")
+    flags+=("--from-password-command=")
+    two_word_flags+=("--from-password-command")
+    local_nonpersistent_flags+=("--from-password-command")
+    local_nonpersistent_flags+=("--from-password-command=")
+    flags+=("--from-password-file=")
+    two_word_flags+=("--from-password-file")
+    local_nonpersistent_flags+=("--from-password-file")
+    local_nonpersistent_flags+=("--from-password-file=")
+    flags+=("--from-repo=")
+    two_word_flags+=("--from-repo")
+    local_nonpersistent_flags+=("--from-repo")
+    local_nonpersistent_flags+=("--from-repo=")
+    flags+=("--from-repository-file=")
+    two_word_flags+=("--from-repository-file")
+    local_nonpersistent_flags+=("--from-repository-file")
+    local_nonpersistent_flags+=("--from-repository-file=")
     flags+=("--help")
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
     local_nonpersistent_flags+=("-h")
-    flags+=("--key-hint2=")
-    two_word_flags+=("--key-hint2")
-    local_nonpersistent_flags+=("--key-hint2")
-    local_nonpersistent_flags+=("--key-hint2=")
-    flags+=("--password-command2=")
-    two_word_flags+=("--password-command2")
-    local_nonpersistent_flags+=("--password-command2")
-    local_nonpersistent_flags+=("--password-command2=")
-    flags+=("--password-file2=")
-    two_word_flags+=("--password-file2")
-    local_nonpersistent_flags+=("--password-file2")
-    local_nonpersistent_flags+=("--password-file2=")
-    flags+=("--repo2=")
-    two_word_flags+=("--repo2")
-    local_nonpersistent_flags+=("--repo2")
-    local_nonpersistent_flags+=("--repo2=")
-    flags+=("--repository-file2=")
-    two_word_flags+=("--repository-file2")
-    local_nonpersistent_flags+=("--repository-file2")
-    local_nonpersistent_flags+=("--repository-file2=")
+    flags+=("--repository-version=")
+    two_word_flags+=("--repository-version")
+    local_nonpersistent_flags+=("--repository-version")
+    local_nonpersistent_flags+=("--repository-version=")
     flags+=("--cacert=")
     two_word_flags+=("--cacert")
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1368,6 +1427,8 @@ _restic_init()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1425,6 +1486,8 @@ _restic_key()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1438,6 +1501,8 @@ _restic_key()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1483,6 +1548,8 @@ _restic_list()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1496,6 +1563,8 @@ _restic_list()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1561,6 +1630,8 @@ _restic_ls()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1574,6 +1645,8 @@ _restic_ls()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1623,6 +1696,8 @@ _restic_migrate()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1636,6 +1711,8 @@ _restic_migrate()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1692,19 +1769,25 @@ _restic_mount()
     two_word_flags+=("--path")
     local_nonpersistent_flags+=("--path")
     local_nonpersistent_flags+=("--path=")
-    flags+=("--snapshot-template=")
-    two_word_flags+=("--snapshot-template")
-    local_nonpersistent_flags+=("--snapshot-template")
-    local_nonpersistent_flags+=("--snapshot-template=")
+    flags+=("--path-template=")
+    two_word_flags+=("--path-template")
+    local_nonpersistent_flags+=("--path-template")
+    local_nonpersistent_flags+=("--path-template=")
     flags+=("--tag=")
     two_word_flags+=("--tag")
     local_nonpersistent_flags+=("--tag")
     local_nonpersistent_flags+=("--tag=")
+    flags+=("--time-template=")
+    two_word_flags+=("--time-template")
+    local_nonpersistent_flags+=("--time-template")
+    local_nonpersistent_flags+=("--time-template=")
     flags+=("--cacert=")
     two_word_flags+=("--cacert")
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1718,6 +1801,8 @@ _restic_mount()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1772,11 +1857,21 @@ _restic_prune()
     local_nonpersistent_flags+=("--max-unused=")
     flags+=("--repack-cacheable-only")
     local_nonpersistent_flags+=("--repack-cacheable-only")
+    flags+=("--repack-small")
+    local_nonpersistent_flags+=("--repack-small")
+    flags+=("--repack-uncompressed")
+    local_nonpersistent_flags+=("--repack-uncompressed")
+    flags+=("--unsafe-recover-no-free-space=")
+    two_word_flags+=("--unsafe-recover-no-free-space")
+    local_nonpersistent_flags+=("--unsafe-recover-no-free-space")
+    local_nonpersistent_flags+=("--unsafe-recover-no-free-space=")
     flags+=("--cacert=")
     two_word_flags+=("--cacert")
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1790,6 +1885,8 @@ _restic_prune()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1837,6 +1934,8 @@ _restic_rebuild-index()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1850,6 +1949,8 @@ _restic_rebuild-index()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1895,6 +1996,8 @@ _restic_recover()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -1908,6 +2011,8 @@ _restic_recover()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -1995,6 +2100,8 @@ _restic_restore()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -2008,6 +2115,8 @@ _restic_restore()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2057,6 +2166,8 @@ _restic_self-update()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -2070,6 +2181,8 @@ _restic_self-update()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2143,6 +2256,8 @@ _restic_snapshots()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -2156,6 +2271,8 @@ _restic_snapshots()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2219,6 +2336,8 @@ _restic_stats()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -2232,6 +2351,8 @@ _restic_stats()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2303,6 +2424,8 @@ _restic_tag()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -2316,6 +2439,8 @@ _restic_tag()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2363,6 +2488,8 @@ _restic_unlock()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -2376,6 +2503,8 @@ _restic_unlock()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2421,6 +2550,8 @@ _restic_version()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
@@ -2434,6 +2565,8 @@ _restic_version()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2502,6 +2635,8 @@ _restic_root_command()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--compression=")
+    two_word_flags+=("--compression")
     flags+=("--help")
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
@@ -2519,6 +2654,8 @@ _restic_root_command()
     flags+=("--option=")
     two_word_flags+=("--option")
     two_word_flags+=("-o")
+    flags+=("--pack-size=")
+    two_word_flags+=("--pack-size")
     flags+=("--password-command=")
     two_word_flags+=("--password-command")
     flags+=("--password-file=")
@@ -2553,6 +2690,7 @@ __start_restic()
     fi
 
     local c=0
+    local flag_parsing_disabled=
     local flags=()
     local two_word_flags=()
     local local_nonpersistent_flags=()
@@ -2562,8 +2700,8 @@ __start_restic()
     local command_aliases=()
     local must_have_one_flag=()
     local must_have_one_noun=()
-    local has_completion_function
-    local last_command
+    local has_completion_function=""
+    local last_command=""
     local nouns=()
     local noun_aliases=()
 

doc/cache.rst

@@ -7,7 +7,7 @@ The location of the cache directory depends on the operating system and the
 environment; see :ref:`caching`.
 
 Each repository has its own cache sub-directory, consisting of the repository ID
-which is chosen at ``init``. All cache directories for different repos are
+which is chosen at ``init``. All cache directories for different repositories are
 independent of each other.
 
 Snapshots, Data and Indexes
@@ -19,8 +19,8 @@ Snapshot, Data and Index files are cached in the sub-directories ``snapshots``,
 Expiry
 ======
 
-Whenever a cache directory for a repo is used, that directory's modification
+Whenever a cache directory for a repository is used, that directory's modification
 timestamp is updated to the current time. By looking at the modification
-timestamps of the repo cache directories it is easy to decide which directories
+timestamps of the repository cache directories it is easy to decide which directories
 are old and haven't been used in a long time. Those are probably stale and can
 be removed.

doc/design.rst

@@ -30,9 +30,10 @@ All data is stored in a restic repository. A repository is able to store
 data of several different types, which can later be requested based on
 an ID. This so-called "storage ID" is the SHA-256 hash of the content of
 a file. All files in a repository are only written once and never
-modified afterwards. This allows accessing and even writing to the
-repository with multiple clients in parallel. Only the ``prune`` operation
-removes data from the repository.
+modified afterwards. Writing should occur atomically to prevent concurrent
+operations from reading incomplete files. This allows accessing and even
+writing to the repository with multiple clients in parallel. Only the ``prune``
+operation removes data from the repository.
 
 Repositories consist of several directories and a top-level file called
 ``config``. For all other files stored in the repository, the name for
@@ -61,28 +62,30 @@ like the following:
 .. code:: json
 
     {
-      "version": 1,
+      "version": 2,
       "id": "5956a3f67a6230d4a92cefb29529f10196c7d92582ec305fd71ff6d331d6271b",
       "chunker_polynomial": "25b468838dcb75"
     }
 
 After decryption, restic first checks that the version field contains a
-version number that it understands, otherwise it aborts. At the moment,
-the version is expected to be 1. The field ``id`` holds a unique ID
-which consists of 32 random bytes, encoded in hexadecimal. This uniquely
-identifies the repository, regardless if it is accessed via SFTP or
-locally. The field ``chunker_polynomial`` contains a parameter that is
-used for splitting large files into smaller chunks (see below).
+version number that it understands, otherwise it aborts. At the moment, the
+version is expected to be 1 or 2. The list of changes in the repository
+format is contained in the section "Changes" below.
+
+The field ``id`` holds a unique ID which consists of 32 random bytes, encoded
+in hexadecimal. This uniquely identifies the repository, regardless if it is
+accessed via a remote storage backend or locally. The field
+``chunker_polynomial`` contains a parameter that is used for splitting large
+files into smaller chunks (see below).
 
 Repository Layout
 -----------------
 
 The ``local`` and ``sftp`` backends are implemented using files and
 directories stored in a file system. The directory layout is the same
-for both backend types.
+for both backend types and is also used for all other remote backends.
 
-The basic layout of a repository stored in a ``local`` or ``sftp``
-backend is shown here:
+The basic layout of a repository is shown here:
 
 ::
 
@@ -108,8 +111,7 @@ backend is shown here:
     │   └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
     └── tmp
 
-A local repository can be initialized with the ``restic init`` command,
-e.g.:
+A local repository can be initialized with the ``restic init`` command, e.g.:
 
 .. code-block:: console
 
@@ -185,40 +187,75 @@ After decryption, a Pack's header consists of the following elements:
 
 ::
 
-    Type_Blob1 || Length(EncryptedBlob1) || Hash(Plaintext_Blob1) ||
+    Type_Blob1 || Data_Blob1 ||
     [...]
-    Type_BlobN || Length(EncryptedBlobN) || Hash(Plaintext_Blobn) ||
+    Type_BlobN || Data_BlobN ||
+
+The Blob type field is a single byte. What follows it depends on the type. The
+following Blob types are defined:
+
++-----------+----------------------+-------------------------------------------------------------------------------+
+| Type      | Meaning              |  Data                                                                         |
++===========+======================+===============================================================================+
+| 0b00      | data blob            |  ``Length(encrypted_blob) || Hash(plaintext_blob)``                           |
++-----------+----------------------+-------------------------------------------------------------------------------+
+| 0b01      | tree blob            |  ``Length(encrypted_blob) || Hash(plaintext_blob)``                           |
++-----------+----------------------+-------------------------------------------------------------------------------+
+| 0b10      | compressed data blob |  ``Length(encrypted_blob) || Length(plaintext_blob) || Hash(plaintext_blob)`` |
++-----------+----------------------+-------------------------------------------------------------------------------+
+| 0b11      | compressed tree blob |  ``Length(encrypted_blob) || Length(plaintext_blob) || Hash(plaintext_blob)`` |
++-----------+----------------------+-------------------------------------------------------------------------------+
 
 This is enough to calculate the offsets for all the Blobs in the Pack.
-Length is the length of a Blob as a four byte integer in little-endian
-format. The type field is a one byte field and labels the content of a
-blob according to the following table:
+The length fields are encoded as four byte integers in little-endian
+format. In the Data column, ``Length(plaintext_blob)`` means the length
+of the decrypted and uncompressed data a blob consists of.
 
-+--------+-----------+
-| Type   | Meaning   |
-+========+===========+
-| 0      | data      |
-+--------+-----------+
-| 1      | tree      |
-+--------+-----------+
+All other types are invalid, more types may be added in the future. The
+compressed types are only valid for repository format version 2. Data and
+tree blobs may be compressed with the zstandard compression algorithm.
 
-All other types are invalid, more types may be added in the future.
+In repository format version 1, data and tree blobs should be stored in
+separate pack files. In version 2, they must be stored in separate files.
+Compressed and non-compress blobs of the same type may be mixed in a pack
+file.
 
 For reconstructing the index or parsing a pack without an index, first
 the last four bytes must be read in order to find the length of the
 header. Afterwards, the header can be read and parsed, which yields all
 plaintext hashes, types, offsets and lengths of all included blobs.
 
+Unpacked Data Format
+====================
+
+Individual files for the index, locks or snapshots are encrypted
+and authenticated like Data and Tree Blobs, so the outer structure is
+``IV || Ciphertext || MAC`` again. In repository format version 1 the
+plaintext always consists of a JSON document which must either be an
+object or an array.
+
+Repository format version 2 adds support for compression. The plaintext
+now starts with a header to indicate the encoding version to distinguish
+it from plain JSON and to allow for further evolution of the storage format:
+``encoding_version || data``
+The ``encoding_version`` field is encoded as one byte.
+For backwards compatibility the encoding versions '[' (0x5b) and '{' (0x7b)
+are used to mark that the whole plaintext (including the encoding version
+byte) should treated as JSON document.
+
+For new data the encoding version is currently always ``2``. For that
+version ``data`` contains a JSON document compressed using the zstandard
+compression algorithm.
+
 Indexing
 ========
 
 Index files contain information about Data and Tree Blobs and the Packs
 they are contained in and store this information in the repository. When
 the local cached index is not accessible any more, the index files can
-be downloaded and used to reconstruct the index. The files are encrypted
-and authenticated like Data and Tree Blobs, so the outer structure is
-``IV || Ciphertext || MAC`` again. The plaintext consists of a JSON
-document like the following:
+be downloaded and used to reconstruct the index. The file encoding is
+described in the "Unpacked Data Format" section. The plaintext consists
+of a JSON document like the following:
 
 .. code:: json
 
@@ -234,18 +271,22 @@ document like the following:
               "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
               "type": "data",
               "offset": 0,
-              "length": 25
-            },{
+              "length": 38,
+              // no 'uncompressed_length' as blob is not compressed
+            },
+            {
               "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
               "type": "tree",
               "offset": 38,
-              "length": 100
+              "length": 112,
+              "uncompressed_length": 511,
             },
             {
               "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
               "type": "data",
               "offset": 150,
-              "length": 123
+              "length": 123,
+              "uncompressed_length": 234,
             }
           ]
         }, [...]
@@ -254,7 +295,11 @@ document like the following:
 
 This JSON document lists Packs and the blobs contained therein. In this
 example, the Pack ``73d04e61`` contains two data Blobs and one Tree
-blob, the plaintext hashes are listed afterwards.
+blob, the plaintext hashes are listed afterwards. The ``length`` field
+corresponds to ``Length(encrypted_blob)`` in the pack file header.
+Field ``uncompressed_length`` is only present for compressed blobs and
+therefore is never present in version 1. It is set to the value of
+``Length(blob)``.
 
 The field ``supersedes`` lists the storage IDs of index files that have
 been replaced with the current index file. This happens when index files
@@ -271,7 +316,7 @@ Keys, Encryption and MAC
 All data stored by restic in the repository is encrypted with AES-256 in
 counter mode and authenticated using Poly1305-AES. For encrypting new
 data first 16 bytes are read from a cryptographically secure
-pseudorandom number generator as a random nonce. This is used both as
+pseudo-random number generator as a random nonce. This is used both as
 the IV for counter mode and the nonce for Poly1305. This operation needs
 three keys: A 32 byte for AES-256 for encryption, a 16 byte AES key and
 a 16 byte key for Poly1305. For details see the original paper `The
@@ -349,8 +394,9 @@ Snapshots
 
 A snapshot represents a directory with all files and sub-directories at
 a given point in time. For each backup that is made, a new snapshot is
-created. A snapshot is a JSON document that is stored in an encrypted
-file below the directory ``snapshots`` in the repository. The filename
+created. A snapshot is a JSON document that is stored in a file below
+the directory ``snapshots`` in the repository. It uses the file encoding
+described in the "Unpacked Data Format" section. The filename
 is the storage ID. This string is unique and used within restic to
 uniquely identify a snapshot.
 
@@ -411,7 +457,7 @@ Blobs of data. The SHA-256 hashes of all Blobs are saved in an ordered
 list which then represents the content of the file.
 
 In order to relate these plaintext hashes to the actual location within
-a Pack file , an index is used. If the index is not available, the
+a Pack file, an index is used. If the index is not available, the
 header of all data Blobs can be read.
 
 Trees and Data
@@ -516,8 +562,8 @@ time there must not be any other locks (exclusive and non-exclusive).
 There may be multiple non-exclusive locks in parallel.
 
 A lock is a file in the subdir ``locks`` whose filename is the storage
-ID of the contents. It is encrypted and authenticated the same way as
-other files in the repository and contains the following JSON structure:
+ID of the contents. It is stored in the file encoding described in the
+"Unpacked Data Format" section and contains the following JSON structure:
 
 .. code:: json
 
@@ -544,6 +590,57 @@ detected, restic creates a new lock, waits, and checks if other locks
 appeared in the repository. Depending on the type of the other locks and
 the lock to be created, restic either continues or fails.
 
+Read and Write Ordering
+=======================
+The repository format allows writing (e.g. backup) and reading (e.g. restore)
+to happen concurrently. As the data for each snapshot in a repository spans
+multiple files (snapshot, index and packs), it is necessary to follow certain
+rules regarding the order in which files are read and written. These ordering
+rules also guarantee that repository modifications always maintain a correct
+repository even if the client or the storage backend crashes for example due
+to a power cut or the (network) connection between both is interrupted.
+
+The correct order to access data in a repository is derived from the following
+set of invariants that must be maintained at **any time** in a correct
+repository. *Must* in the following is a strict requirement and will lead to
+data loss if not followed. *Should* will require steps to fix a repository
+(e.g. rebuilding the index) if not followed, but should not cause data loss.
+*existing* means that the referenced data is **durably** stored in the repository.
+
+- A snapshot *must* only reference an existing tree blob.
+- A reachable tree blob *must* only reference tree and data blobs that exist
+  (recursively). *Reachable* means that the tree blob is reachable starting from
+  a snapshot.
+- An index *must* only reference valid blobs in existing packs.
+- All blobs referenced by a snapshot *should* be listed in an index.
+
+This leads to the following recommended order to store data in a repository.
+First, pack files, which contain data and tree blobs, must be written. Then the
+indexes which reference blobs in these already written pack files. And finally
+the corresponding snapshots.
+
+Note that there is no need for a specific write order of data and tree blobs
+during a backup as the blobs only become referenced once the corresponding
+snapshot is uploaded.
+
+Reading data should follow the opposite order compared to writing. Only once a
+snapshot was written, it is guaranteed that all required data exists in the
+repository. This especially means that the list of snapshots to read should be
+collected before loading the repository index. The other way round can lead to
+a race condition where a recently written snapshot is loaded but not its
+accompanying index, which results in a failure to access the snapshot's tree
+blob.
+
+For removing or rewriting data from a repository the following rules must be
+followed, which are derived from the above invariants.
+
+- A client removing data *must* acquire an exclusive lock first to prevent
+  conflicts with other clients.
+- A pack *must* be removed from the referencing index before it is deleted.
+- Rewriting a pack *must* write the new pack, update the index (add an updated
+  index and delete the old one) and only then delete the old pack.
+
+
 Backups and Deduplication
 =========================
 
@@ -584,10 +681,10 @@ General assumptions:
    key management design, it is impossible to securely revoke a leaked key
    without re-encrypting the whole repository.
 -  Advances in cryptography attacks against the cryptographic primitives used
-   by restic (i.e, AES-256-CTR-Poly1305-AES and SHA-256) have not occurred. Such
+   by restic (i.e., AES-256-CTR-Poly1305-AES and SHA-256) have not occurred. Such
    advances could render the confidentiality or integrity protections provided
    by restic useless.
--  Sufficient advances in computing have not occurred to make bruteforce
+-  Sufficient advances in computing have not occurred to make brute-force
    attacks against restic's cryptographic protections feasible.
 
 The restic backup program guarantees the following:
@@ -669,3 +766,11 @@ An adversary who has a leaked (decrypted) key for a repository could:
    only be done using the ``copy`` command, which moves the data into a new
    repository with a new master key, or by making a completely new repository
    and new backup.
+
+Changes
+=======
+
+Repository Version 2
+--------------------
+
+ * Support compression for blobs (data/tree) and index / lock / snapshot files

doc/faq.rst

@@ -51,7 +51,7 @@ looks like this:
     [0:00] 100.00%  16 / 16 snapshots
     no errors were found
 
-The message means that there is more data stored in the repo than
+The message means that there is more data stored in the repository than
 strictly necessary. This is uncritical. With high probability this is duplicate data
 caused by an interrupted backup run or upload operation. In
 order to clean it up, the command ``restic prune`` can be used.
@@ -168,8 +168,8 @@ scheduling algorithm to give it the least favorable niceness (19).
 The above example makes sure that the system the backup runs on
 is not slowed down, which is particularly useful for servers.
 
-Creating new repo on a Synology NAS via sftp fails
---------------------------------------------------
+Creating new repository on a Synology NAS via sftp fails
+--------------------------------------------------------
 
 For using restic with a Synology NAS via sftp, please make sure that the
 specified path is absolute, it must start with a slash (``/``).

doc/fish-completion.fish

@@ -18,7 +18,8 @@ function __restic_perform_completion
     __restic_debug "args: $args"
     __restic_debug "last arg: $lastArg"
 
-    set -l requestComp "$args[1] __complete $args[2..-1] $lastArg"
+    # Disable ActiveHelp which is not supported for fish shell
+    set -l requestComp "RESTIC_ACTIVE_HELP=0 $args[1] __complete $args[2..-1] $lastArg"
 
     __restic_debug "Calling $requestComp"
     set -l results (eval $requestComp 2> /dev/null)

doc/index.rst

@@ -9,6 +9,7 @@ Restic Documentation
    030_preparing_a_new_repo
    040_backup
    045_working_with_repos
+   047_tuning_backup_parameters
    050_restore
    060_forget
    070_encryption

doc/man/restic-backup.1

@@ -3,12 +3,12 @@
 
 .SH NAME
 .PP
-restic\-backup \- Create a new backup of files and/or directories
+restic-backup - Create a new backup of files and/or directories
 
 
 .SH SYNOPSIS
 .PP
-\fBrestic backup [flags] FILE/DIR [FILE/DIR] ...\fP
+\fBrestic backup [flags] [FILE/DIR] ...\fP
 
 
 .SH DESCRIPTION
@@ -26,170 +26,178 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 
 .SH OPTIONS
 .PP
-\fB\-n\fP, \fB\-\-dry\-run\fP[=false]
+\fB-n\fP, \fB--dry-run\fP[=false]
 	do not upload or write any data, just show what would be done
 
 .PP
-\fB\-e\fP, \fB\-\-exclude\fP=[]
+\fB-e\fP, \fB--exclude\fP=[]
 	exclude a \fB\fCpattern\fR (can be specified multiple times)
 
 .PP
-\fB\-\-exclude\-caches\fP[=false]
+\fB--exclude-caches\fP[=false]
 	excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard
 
 .PP
-\fB\-\-exclude\-file\fP=[]
+\fB--exclude-file\fP=[]
 	read exclude patterns from a \fB\fCfile\fR (can be specified multiple times)
 
 .PP
-\fB\-\-exclude\-if\-present\fP=[]
+\fB--exclude-if-present\fP=[]
 	takes \fB\fCfilename[:header]\fR, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)
 
 .PP
-\fB\-\-exclude\-larger\-than\fP=""
+\fB--exclude-larger-than\fP=""
 	max \fB\fCsize\fR of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)
 
 .PP
-\fB\-\-files\-from\fP=[]
+\fB--files-from\fP=[]
 	read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times)
 
 .PP
-\fB\-\-files\-from\-raw\fP=[]
+\fB--files-from-raw\fP=[]
 	read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times)
 
 .PP
-\fB\-\-files\-from\-verbatim\fP=[]
+\fB--files-from-verbatim\fP=[]
 	read the files to backup from \fB\fCfile\fR (can be combined with file args; can be specified multiple times)
 
 .PP
-\fB\-f\fP, \fB\-\-force\fP[=false]
-	force re\-reading the target files/directories (overrides the "parent" flag)
+\fB-f\fP, \fB--force\fP[=false]
+	force re-reading the target files/directories (overrides the "parent" flag)
 
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB-h\fP, \fB--help\fP[=false]
 	help for backup
 
 .PP
-\fB\-H\fP, \fB\-\-host\fP=""
+\fB-H\fP, \fB--host\fP=""
 	set the \fB\fChostname\fR for the snapshot manually. To prevent an expensive rescan use the "parent" flag
 
 .PP
-\fB\-\-iexclude\fP=[]
-	same as \-\-exclude \fB\fCpattern\fR but ignores the casing of filenames
+\fB--iexclude\fP=[]
+	same as --exclude \fB\fCpattern\fR but ignores the casing of filenames
 
 .PP
-\fB\-\-iexclude\-file\fP=[]
-	same as \-\-exclude\-file but ignores casing of \fB\fCfile\fRnames in patterns
+\fB--iexclude-file\fP=[]
+	same as --exclude-file but ignores casing of \fB\fCfile\fRnames in patterns
 
 .PP
-\fB\-\-ignore\-ctime\fP[=false]
+\fB--ignore-ctime\fP[=false]
 	ignore ctime changes when checking for modified files
 
 .PP
-\fB\-\-ignore\-inode\fP[=false]
+\fB--ignore-inode\fP[=false]
 	ignore inode number changes when checking for modified files
 
 .PP
-\fB\-x\fP, \fB\-\-one\-file\-system\fP[=false]
+\fB-x\fP, \fB--one-file-system\fP[=false]
 	exclude other file systems, don't cross filesystem boundaries and subvolumes
 
 .PP
-\fB\-\-parent\fP=""
-	use this parent \fB\fCsnapshot\fR (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)
+\fB--parent\fP=""
+	use this parent \fB\fCsnapshot\fR (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)
 
 .PP
-\fB\-\-stdin\fP[=false]
+\fB--stdin\fP[=false]
 	read backup from stdin
 
 .PP
-\fB\-\-stdin\-filename\fP="stdin"
+\fB--stdin-filename\fP="stdin"
 	\fB\fCfilename\fR to use when reading from stdin
 
 .PP
-\fB\-\-tag\fP=[]
+\fB--tag\fP=[]
 	add \fB\fCtags\fR for the new snapshot in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times)
 
 .PP
-\fB\-\-time\fP=""
-	\fB\fCtime\fR of the backup (ex. '2012\-11\-01 22:08:41') (default: now)
+\fB--time\fP=""
+	\fB\fCtime\fR of the backup (ex. '2012-11-01 22:08:41') (default: now)
 
 .PP
-\fB\-\-with\-atime\fP[=false]
+\fB--with-atime\fP[=false]
 	store the atime for all files and directories
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO

doc/man/restic-cache.1

@@ -3,7 +3,7 @@
 
 .SH NAME
 .PP
-restic\-cache \- Operate on local cache directories
+restic-cache - Operate on local cache directories
 
 
 .SH SYNOPSIS
@@ -18,99 +18,107 @@ The "cache" command allows listing and cleaning local cache directories.
 
 .SH EXIT STATUS
 .PP
-Exit status is 0 if the command was successful, and non\-zero if there was any error.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 
 
 .SH OPTIONS
 .PP
-\fB\-\-cleanup\fP[=false]
+\fB--cleanup\fP[=false]
 	remove old cache directories
 
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB-h\fP, \fB--help\fP[=false]
 	help for cache
 
 .PP
-\fB\-\-max\-age\fP=30
+\fB--max-age\fP=30
 	max age in \fB\fCdays\fR for cache directories to be considered old
 
 .PP
-\fB\-\-no\-size\fP[=false]
+\fB--no-size\fP[=false]
 	do not output the size of the cache directories
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO

doc/man/restic-cat.1

@@ -3,7 +3,7 @@
 
 .SH NAME
 .PP
-restic\-cat \- Print internal objects to stdout
+restic-cat - Print internal objects to stdout
 
 
 .SH SYNOPSIS
@@ -18,87 +18,95 @@ The "cat" command is used to print internal objects to stdout.
 
 .SH EXIT STATUS
 .PP
-Exit status is 0 if the command was successful, and non\-zero if there was any error.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 
 
 .SH OPTIONS
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB-h\fP, \fB--help\fP[=false]
 	help for cat
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO

doc/man/restic-check.1

@@ -3,7 +3,7 @@
 
 .SH NAME
 .PP
-restic\-check \- Check the repository for errors
+restic-check - Check the repository for errors
 
 
 .SH SYNOPSIS
@@ -23,103 +23,107 @@ repository and not use a local cache.
 
 .SH EXIT STATUS
 .PP
-Exit status is 0 if the command was successful, and non\-zero if there was any error.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 
 
 .SH OPTIONS
 .PP
-\fB\-\-check\-unused\fP[=false]
-	find unused blobs
-
-.PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB-h\fP, \fB--help\fP[=false]
 	help for check
 
 .PP
-\fB\-\-read\-data\fP[=false]
+\fB--read-data\fP[=false]
 	read all data blobs
 
 .PP
-\fB\-\-read\-data\-subset\fP=""
+\fB--read-data-subset\fP=""
 	read a \fB\fCsubset\fR of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset
 
 .PP
-\fB\-\-with\-cache\fP[=false]
+\fB--with-cache\fP[=false]
 	use the cache
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO

doc/man/restic-copy.1

@@ -3,7 +3,7 @@
 
 .SH NAME
 .PP
-restic\-copy \- Copy snapshots from one repository to another
+restic-copy - Copy snapshots from one repository to another
 
 
 .SH SYNOPSIS
@@ -22,124 +22,132 @@ destination repositories. This /may incur higher bandwidth usage and costs/ than
 expected during normal backup runs.
 
 .PP
-NOTE: The copying process does not re\-chunk files, which may break deduplication
+NOTE: The copying process does not re-chunk files, which may break deduplication
 between the files copied and files already stored in the destination repository.
 This means that copied files, which existed in both the source and destination
 repository, /may occupy up to twice their space/ in the destination repository.
-This can be mitigated by the "\-\-copy\-chunker\-params" option when initializing a
+This can be mitigated by the "--copy-chunker-params" option when initializing a
 new destination repository using the "init" command.
 
 
 .SH OPTIONS
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB--from-key-hint\fP=""
+	key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)
+
+.PP
+\fB--from-password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)
+
+.PP
+\fB--from-password-file\fP=""
+	\fB\fCfile\fR to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)
+
+.PP
+\fB--from-repo\fP=""
+	source \fB\fCrepository\fR to copy snapshots from (default: $RESTIC_FROM_REPOSITORY)
+
+.PP
+\fB--from-repository-file\fP=""
+	\fB\fCfile\fR from which to read the source repository location to copy snapshots from (default: $RESTIC_FROM_REPOSITORY_FILE)
+
+.PP
+\fB-h\fP, \fB--help\fP[=false]
 	help for copy
 
 .PP
-\fB\-H\fP, \fB\-\-host\fP=[]
+\fB-H\fP, \fB--host\fP=[]
 	only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times)
 
 .PP
-\fB\-\-key\-hint2\fP=""
-	key ID of key to try decrypting the destination repository first (default: $RESTIC\_KEY\_HINT2)
-
-.PP
-\fB\-\-password\-command2\fP=""
-	shell \fB\fCcommand\fR to obtain the destination repository password from (default: $RESTIC\_PASSWORD\_COMMAND2)
-
-.PP
-\fB\-\-password\-file2\fP=""
-	\fB\fCfile\fR to read the destination repository password from (default: $RESTIC\_PASSWORD\_FILE2)
-
-.PP
-\fB\-\-path\fP=[]
+\fB--path\fP=[]
 	only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot ID is given
 
 .PP
-\fB\-\-repo2\fP=""
-	destination \fB\fCrepository\fR to copy snapshots to (default: $RESTIC\_REPOSITORY2)
-
-.PP
-\fB\-\-repository\-file2\fP=""
-	\fB\fCfile\fR from which to read the destination repository location to copy snapshots to (default: $RESTIC\_REPOSITORY\_FILE2)
-
-.PP
-\fB\-\-tag\fP=[]
+\fB--tag\fP=[]
 	only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot ID is given
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO

doc/man/restic-diff.1

@@ -3,12 +3,12 @@
 
 .SH NAME
 .PP
-restic\-diff \- Show differences between two snapshots
+restic-diff - Show differences between two snapshots
 
 
 .SH SYNOPSIS
 .PP
-\fBrestic diff [flags] snapshot\-ID snapshot\-ID\fP
+\fBrestic diff [flags] snapshot-ID snapshot-ID\fP
 
 
 .SH DESCRIPTION
@@ -21,7 +21,7 @@ directory:
 .IP \(bu 2
 +  The item was added
 .IP \(bu 2
-\-  The item was removed
+-  The item was removed
 .IP \(bu 2
 U  The metadata (access mode, timestamps, ...) for the item was updated
 .IP \(bu 2
@@ -34,91 +34,99 @@ T  The type was changed, e.g. a file was made a symlink
 
 .SH EXIT STATUS
 .PP
-Exit status is 0 if the command was successful, and non\-zero if there was any error.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 
 
 .SH OPTIONS
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB-h\fP, \fB--help\fP[=false]
 	help for diff
 
 .PP
-\fB\-\-metadata\fP[=false]
+\fB--metadata\fP[=false]
 	print changes in metadata
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO

doc/man/restic-dump.1

@@ -3,7 +3,7 @@
 
 .SH NAME
 .PP
-restic\-dump \- Print a backed\-up file to stdout
+restic-dump - Print a backed-up file to stdout
 
 
 .SH SYNOPSIS
@@ -25,103 +25,111 @@ repository.
 
 .SH EXIT STATUS
 .PP
-Exit status is 0 if the command was successful, and non\-zero if there was any error.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 
 
 .SH OPTIONS
 .PP
-\fB\-a\fP, \fB\-\-archive\fP="tar"
+\fB-a\fP, \fB--archive\fP="tar"
 	set archive \fB\fCformat\fR as "tar" or "zip"
 
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB-h\fP, \fB--help\fP[=false]
 	help for dump
 
 .PP
-\fB\-H\fP, \fB\-\-host\fP=[]
+\fB-H\fP, \fB--host\fP=[]
 	only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)
 
 .PP
-\fB\-\-path\fP=[]
+\fB--path\fP=[]
 	only consider snapshots which include this (absolute) \fB\fCpath\fR for snapshot ID "latest"
 
 .PP
-\fB\-\-tag\fP=[]
+\fB--tag\fP=[]
 	only consider snapshots which include this \fB\fCtaglist\fR for snapshot ID "latest"
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO

doc/man/restic-find.1

@@ -3,7 +3,7 @@
 
 .SH NAME
 .PP
-restic\-find \- Find a file, a directory or restic IDs
+restic-find - Find a file, a directory or restic IDs
 
 
 .SH SYNOPSIS
@@ -20,130 +20,138 @@ It can also be used to search for restic blobs or trees for troubleshooting.
 
 .SH OPTIONS
 .PP
-\fB\-\-blob\fP[=false]
-	pattern is a blob\-ID
+\fB--blob\fP[=false]
+	pattern is a blob-ID
 
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB-h\fP, \fB--help\fP[=false]
 	help for find
 
 .PP
-\fB\-H\fP, \fB\-\-host\fP=[]
+\fB-H\fP, \fB--host\fP=[]
 	only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times)
 
 .PP
-\fB\-i\fP, \fB\-\-ignore\-case\fP[=false]
+\fB-i\fP, \fB--ignore-case\fP[=false]
 	ignore case for pattern
 
 .PP
-\fB\-l\fP, \fB\-\-long\fP[=false]
+\fB-l\fP, \fB--long\fP[=false]
 	use a long listing format showing size and mode
 
 .PP
-\fB\-N\fP, \fB\-\-newest\fP=""
+\fB-N\fP, \fB--newest\fP=""
 	newest modification date/time
 
 .PP
-\fB\-O\fP, \fB\-\-oldest\fP=""
+\fB-O\fP, \fB--oldest\fP=""
 	oldest modification date/time
 
 .PP
-\fB\-\-pack\fP[=false]
-	pattern is a pack\-ID
+\fB--pack\fP[=false]
+	pattern is a pack-ID
 
 .PP
-\fB\-\-path\fP=[]
-	only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot\-ID is given
+\fB--path\fP=[]
+	only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot-ID is given
 
 .PP
-\fB\-\-show\-pack\-id\fP[=false]
-	display the pack\-ID the blobs belong to (with \-\-blob or \-\-tree)
+\fB--show-pack-id\fP[=false]
+	display the pack-ID the blobs belong to (with --blob or --tree)
 
 .PP
-\fB\-s\fP, \fB\-\-snapshot\fP=[]
+\fB-s\fP, \fB--snapshot\fP=[]
 	snapshot \fB\fCid\fR to search in (can be given multiple times)
 
 .PP
-\fB\-\-tag\fP=[]
-	only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot\-ID is given
+\fB--tag\fP=[]
+	only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot-ID is given
 
 .PP
-\fB\-\-tree\fP[=false]
-	pattern is a tree\-ID
+\fB--tree\fP[=false]
+	pattern is a tree-ID
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH EXAMPLE
@@ -152,16 +160,16 @@ It can also be used to search for restic blobs or trees for troubleshooting.
 
 .nf
 restic find config.json
-restic find \-\-json "*.yml" "*.json"
-restic find \-\-json \-\-blob 420f620f b46ebe8a ddd38656
-restic find \-\-show\-pack\-id \-\-blob 420f620f
-restic find \-\-tree 577c2bc9 f81f2e22 a62827a9
-restic find \-\-pack 025c1d06
+restic find --json "*.yml" "*.json"
+restic find --json --blob 420f620f b46ebe8a ddd38656
+restic find --show-pack-id --blob 420f620f
+restic find --tree 577c2bc9 f81f2e22 a62827a9
+restic find --pack 025c1d06
 
 EXIT STATUS
 ===========
 
-Exit status is 0 if the command was successful, and non\-zero if there was any error.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 
 
 .fi

doc/man/restic-forget.1

@@ -3,7 +3,7 @@
 
 .SH NAME
 .PP
-restic\-forget \- Remove snapshots from the repository
+restic-forget - Remove snapshots from the repository
 
 
 .SH SYNOPSIS
@@ -13,188 +13,211 @@ restic\-forget \- Remove snapshots from the repository
 
 .SH DESCRIPTION
 .PP
-The "forget" command removes snapshots according to a policy. Please note that
-this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove the unreferenced data
-after "forget" was run successfully, see the "prune" command. Please also read
-the documentation for "forget" to learn about important security considerations.
+The "forget" command removes snapshots according to a policy. All snapshots are
+first divided into groups according to "--group-by", and after that the policy
+specified by the "--keep-*" options is applied to each group individually.
+
+.PP
+Please note that this command really only deletes the snapshot object in the
+repository, which is a reference to data stored there. In order to remove the
+unreferenced data after "forget" was run successfully, see the "prune" command.
+
+.PP
+Please also read the documentation for "forget" to learn about some important
+security considerations.
 
 
 .SH EXIT STATUS
 .PP
-Exit status is 0 if the command was successful, and non\-zero if there was any error.
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 
 
 .SH OPTIONS
 .PP
-\fB\-l\fP, \fB\-\-keep\-last\fP=0
+\fB-l\fP, \fB--keep-last\fP=0
 	keep the last \fB\fCn\fR snapshots
 
 .PP
-\fB\-H\fP, \fB\-\-keep\-hourly\fP=0
+\fB-H\fP, \fB--keep-hourly\fP=0
 	keep the last \fB\fCn\fR hourly snapshots
 
 .PP
-\fB\-d\fP, \fB\-\-keep\-daily\fP=0
+\fB-d\fP, \fB--keep-daily\fP=0
 	keep the last \fB\fCn\fR daily snapshots
 
 .PP
-\fB\-w\fP, \fB\-\-keep\-weekly\fP=0
+\fB-w\fP, \fB--keep-weekly\fP=0
 	keep the last \fB\fCn\fR weekly snapshots
 
 .PP
-\fB\-m\fP, \fB\-\-keep\-monthly\fP=0
+\fB-m\fP, \fB--keep-monthly\fP=0
 	keep the last \fB\fCn\fR monthly snapshots
 
 .PP
-\fB\-y\fP, \fB\-\-keep\-yearly\fP=0
+\fB-y\fP, \fB--keep-yearly\fP=0
 	keep the last \fB\fCn\fR yearly snapshots
 
 .PP
-\fB\-\-keep\-within\fP=
+\fB--keep-within\fP=
 	keep snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot
 
 .PP
-\fB\-\-keep\-within\-hourly\fP=
+\fB--keep-within-hourly\fP=
 	keep hourly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot
 
 .PP
-\fB\-\-keep\-within\-daily\fP=
+\fB--keep-within-daily\fP=
 	keep daily snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot
 
 .PP
-\fB\-\-keep\-within\-weekly\fP=
+\fB--keep-within-weekly\fP=
 	keep weekly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot
 
 .PP
-\fB\-\-keep\-within\-monthly\fP=
+\fB--keep-within-monthly\fP=
 	keep monthly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot
 
 .PP
-\fB\-\-keep\-within\-yearly\fP=
+\fB--keep-within-yearly\fP=
 	keep yearly snapshots that are newer than \fB\fCduration\fR (eg. 1y5m7d2h) relative to the latest snapshot
 
 .PP
-\fB\-\-keep\-tag\fP=[]
+\fB--keep-tag\fP=[]
 	keep snapshots with this \fB\fCtaglist\fR (can be specified multiple times)
 
 .PP
-\fB\-\-host\fP=[]
+\fB--host\fP=[]
 	only consider snapshots with the given \fB\fChost\fR (can be specified multiple times)
 
 .PP
-\fB\-\-tag\fP=[]
+\fB--tag\fP=[]
 	only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times)
 
 .PP
-\fB\-\-path\fP=[]
+\fB--path\fP=[]
 	only consider snapshots which include this (absolute) \fB\fCpath\fR (can be specified multiple times)
 
 .PP
-\fB\-c\fP, \fB\-\-compact\fP[=false]
+\fB-c\fP, \fB--compact\fP[=false]
 	use compact output format
 
 .PP
-\fB\-g\fP, \fB\-\-group\-by\fP="host,paths"
-	string for grouping snapshots by host,paths,tags
+\fB-g\fP, \fB--group-by\fP="host,paths"
+	\fB\fCgroup\fR snapshots by host, paths and/or tags, separated by comma (disable grouping with '')
 
 .PP
-\fB\-n\fP, \fB\-\-dry\-run\fP[=false]
+\fB-n\fP, \fB--dry-run\fP[=false]
 	do not delete anything, just print what would be done
 
 .PP
-\fB\-\-prune\fP[=false]
+\fB--prune\fP[=false]
 	automatically run the 'prune' command if snapshots have been removed
 
 .PP
-\fB\-\-max\-unused\fP="5%"
+\fB--max-unused\fP="5%"
 	tolerate given \fB\fClimit\fR of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')
 
 .PP
-\fB\-\-max\-repack\-size\fP=""
+\fB--max-repack-size\fP=""
 	maximum \fB\fCsize\fR to repack (allowed suffixes: k/K, m/M, g/G, t/T)
 
 .PP
-\fB\-\-repack\-cacheable\-only\fP[=false]
+\fB--repack-cacheable-only\fP[=false]
 	only repack packs which are cacheable
 
 .PP
-\fB\-h\fP, \fB\-\-help\fP[=false]
+\fB--repack-small\fP[=false]
+	repack pack files below 80% of target pack size
+
+.PP
+\fB--repack-uncompressed\fP[=false]
+	repack all uncompressed data
+
+.PP
+\fB-h\fP, \fB--help\fP[=false]
 	help for forget
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
 .PP
-\fB\-\-cacert\fP=[]
+\fB--cacert\fP=[]
 	\fB\fCfile\fR to load root certificates from (default: use system certificates)
 
 .PP
-\fB\-\-cache\-dir\fP=""
+\fB--cache-dir\fP=""
 	set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
 
 .PP
-\fB\-\-cleanup\-cache\fP[=false]
+\fB--cleanup-cache\fP[=false]
 	auto remove old cache directories
 
 .PP
-\fB\-\-insecure\-tls\fP[=false]
-	skip TLS certificate verification when connecting to the repo (insecure)
+\fB--compression\fP=auto
+	compression mode (only available for repository format version 2), one of (auto|off|max)
 
 .PP
-\fB\-\-json\fP[=false]
+\fB--insecure-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
 	set output mode to JSON for commands that support it
 
 .PP
-\fB\-\-key\-hint\fP=""
-	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC\_KEY\_HINT)
+\fB--key-hint\fP=""
+	\fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
 
 .PP
-\fB\-\-limit\-download\fP=0
+\fB--limit-download\fP=0
 	limits downloads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-limit\-upload\fP=0
+\fB--limit-upload\fP=0
 	limits uploads to a maximum rate in KiB/s. (default: unlimited)
 
 .PP
-\fB\-\-no\-cache\fP[=false]
+\fB--no-cache\fP[=false]
 	do not use a local cache
 
 .PP
-\fB\-\-no\-lock\fP[=false]
-	do not lock the repository, this allows some operations on read\-only repositories
+\fB--no-lock\fP[=false]
+	do not lock the repository, this allows some operations on read-only repositories
 
 .PP
-\fB\-o\fP, \fB\-\-option\fP=[]
+\fB-o\fP, \fB--option\fP=[]
 	set extended option (\fB\fCkey=value\fR, can be specified multiple times)
 
 .PP
-\fB\-\-password\-command\fP=""
-	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC\_PASSWORD\_COMMAND)
+\fB--pack-size\fP=0
+	set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
 
 .PP
-\fB\-p\fP, \fB\-\-password\-file\fP=""
-	\fB\fCfile\fR to read the repository password from (default: $RESTIC\_PASSWORD\_FILE)
+\fB--password-command\fP=""
+	shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
 
 .PP
-\fB\-q\fP, \fB\-\-quiet\fP[=false]
+\fB-p\fP, \fB--password-file\fP=""
+	\fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
 	do not output comprehensive progress report
 
 .PP
-\fB\-r\fP, \fB\-\-repo\fP=""
-	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC\_REPOSITORY)
+\fB-r\fP, \fB--repo\fP=""
+	\fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 .PP
-\fB\-\-repository\-file\fP=""
-	\fB\fCfile\fR to read the repository location from (default: $RESTIC\_REPOSITORY\_FILE)
+\fB--repository-file\fP=""
+	\fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
 
 .PP
-\fB\-\-tls\-client\-cert\fP=""
+\fB--tls-client-cert\fP=""
 	path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
 
 .PP
-\fB\-v\fP, \fB\-\-verbose\fP[=0]
-	be verbose (specify multiple times or a level using \-\-verbose=\fB\fCn\fR, max level/times is 3)
+\fB-v\fP, \fB--verbose\fP[=0]
+	be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
 
 
 .SH SEE ALSO