Add version for 0.13.1

Fixes #3687. Uses the cast suggested by @MichaelEischer, except that the
contant isn't cast along, because it's untyped and will be converted by
the compiler as necessary.

.dockerignore

@@ -0,0 +1,12 @@
+# Folders
+.git/
+.github/
+changelog/
+doc/
+docker/
+helpers/
+
+# Files
+.gitignore
+.golangci.yml
+*.md

.github/workflows/tests.yml

@@ -8,6 +8,10 @@ on:
   # run tests for all pull requests
   pull_request:
 
+env:
+  latest_go: "1.18.x"
+  GO111MODULE: on
+
 jobs:
   test:
     strategy:
@@ -15,35 +19,47 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.16.x
+            go: 1.18.x
             os: windows-latest
+            install_verb: install
 
           - job_name: macOS
-            go: 1.16.x
+            go: 1.18.x
             os: macOS-latest
             test_fuse: false
+            install_verb: install
 
           - job_name: Linux
-            go: 1.16.x
+            go: 1.18.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
+            install_verb: install
+
+          - job_name: Linux
+            go: 1.17.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: install
+
+          - job_name: Linux
+            go: 1.16.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: get
 
           - job_name: Linux
             go: 1.15.x
             os: ubuntu-latest
             test_fuse: true
+            install_verb: get
 
           - job_name: Linux
             go: 1.14.x
             os: ubuntu-latest
             test_fuse: true
-
-          - job_name: Linux
-            go: 1.13.x
-            os: ubuntu-latest
-            test_fuse: true
+            install_verb: get
 
     name: ${{ matrix.job_name }} Go ${{ matrix.go }}
     runs-on: ${{ matrix.os }}
@@ -60,7 +76,7 @@ jobs:
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go get github.com/restic/rest-server/...
+          go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -92,7 +108,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go get github.com/restic/rest-server/...
+          go ${{ matrix.install_verb }} github.com/restic/rest-server/...
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -182,7 +198,7 @@ jobs:
       - name: Check changelog files with calens
         run: |
           echo "install calens"
-          go get github.com/restic/calens
+          go install github.com/restic/calens@latest
 
           echo "check changelog files"
           calens
@@ -206,7 +222,6 @@ jobs:
             solaris/amd64"
 
     env:
-      go: 1.16.x
       GOPROXY: https://proxy.golang.org
 
     runs-on: ubuntu-latest
@@ -214,17 +229,17 @@ jobs:
     name: Cross Compile for ${{ matrix.targets }}
 
     steps:
-      - name: Set up Go ${{ env.go }}
+      - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v2
         with:
-          go-version: ${{ env.go }}
-
-      - name: Check out code
-        uses: actions/checkout@v2
+          go-version: ${{ env.latest_go }}
 
       - name: Install gox
         run: |
-          go get github.com/mitchellh/gox
+          go install github.com/mitchellh/gox@latest
+
+      - name: Check out code
+        uses: actions/checkout@v2
 
       - name: Cross-compile with gox for ${{ matrix.targets }}
         env:
@@ -238,13 +253,11 @@ jobs:
   lint:
     name: lint
     runs-on: ubuntu-latest
-    env:
-      go: 1.16.x
     steps:
-      - name: Set up Go ${{ env.go }}
+      - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v2
         with:
-          go-version: ${{ env.go }}
+          go-version: ${{ env.latest_go }}
 
       - name: Check out code
         uses: actions/checkout@v2
@@ -253,7 +266,7 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.36
+          version: v1.45
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           only-new-issues: true
           args: --verbose --timeout 5m
@@ -269,3 +282,44 @@ jobs:
           echo "check if go.mod and go.sum are up to date"
           go mod tidy
           git diff --exit-code go.mod go.sum
+
+  docker:
+    name: docker
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            restic/restic
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: false
+          context: .
+          file: docker/Dockerfile
+          pull: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

CHANGELOG.md

@@ -1,3 +1,444 @@
+Changelog for restic 0.13.1 (2022-04-10)
+=======================================
+
+The following sections list the changes in restic 0.13.1 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #3685: Fix the diff command
+ * Fix #3681: Fix rclone (shimmed by Scoop) and sftp stopped working on Windows
+
+Details
+-------
+
+ * Bugfix #3685: Fix the diff command
+
+   There was a bug in the `diff` command, it would always show files in a removed directory as added.
+   We've fixed that.
+
+   https://github.com/restic/restic/issues/3685
+   https://github.com/restic/restic/pull/3686
+
+ * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp stopped working on Windows
+
+   In #3602 a fix was introduced to fix the problem that rclone prematurely exits when Ctrl+C is
+   pressed on Windows. The solution was to create the subprocess with its console detached from
+   the restic console. However, such solution fails when using rclone install by scoop or using
+   sftp with a passphrase- protected private key. We've fixed that by using a different approach
+   to prevent Ctrl-C from passing down too early.
+
+   https://github.com/restic/restic/issues/3681
+   https://github.com/restic/restic/issues/3692
+   https://github.com/restic/restic/pull/3696
+
+
+Changelog for restic 0.13.0 (2022-03-26)
+=======================================
+
+The following sections list the changes in restic 0.13.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1106: Never lock repository for `list locks`
+ * Fix #2345: Make cache crash-resistant and usable by multiple concurrent processes
+ * Fix #2452: Improve error handling of repository locking
+ * Fix #2738: Don't print progress for `backup --json --quiet`
+ * Fix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+ * Fix #3518: Make `copy` command honor `--no-lock` for source repository
+ * Fix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
+ * Fix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+ * Fix #3667: The `mount` command now reports symlinks sizes
+ * Fix #3488: `rebuild-index` failed if an index file was damaged
+ * Fix #3591: Fix handling of `prune --max-repack-size=0`
+ * Fix #3619: Avoid choosing parent snapshots newer than time of new snapshot
+ * Chg #3641: Ignore parent snapshot for `backup --stdin`
+ * Chg #3519: Require Go 1.14 or newer
+ * Enh #1542: Add `--dry-run`/`-n` option to `backup` command
+ * Enh #2202: Add upload checksum for Azure, GS, S3 and Swift backends
+ * Enh #233: Support negative include/exclude patterns
+ * Enh #2388: Add warning for S3 if partial credentials are provided
+ * Enh #2508: Support JSON output and quiet mode for the `diff` command
+ * Enh #2656: Add flag to disable TLS verification for self-signed certificates
+ * Enh #3003: Atomic uploads for the SFTP backend
+ * Enh #3127: Add xattr (extended attributes) support for Solaris
+ * Enh #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+ * Enh #3490: Support random subset by size in `check --read-data-subset`
+ * Enh #3541: Improve handling of temporary B2 delete errors
+ * Enh #3542: Add file mode in symbolic notation to `ls --json`
+ * Enh #2594: Speed up the `restore --verify` command
+ * Enh #2816: The `backup` command no longer updates file access times on Linux
+ * Enh #2880: Make `recover` collect only unreferenced trees
+ * Enh #3429: Verify that new or modified keys are stored correctly
+ * Enh #3436: Improve local backend's resilience to (system) crashes
+ * Enh #3508: Cache blobs read by the `dump` command
+ * Enh #3511: Support configurable timeout for the rclone backend
+ * Enh #3593: Improve `copy` performance by parallelizing IO
+
+Details
+-------
+
+ * Bugfix #1106: Never lock repository for `list locks`
+
+   The `list locks` command previously locked to the repository by default. This had the problem
+   that it wouldn't work for an exclusively locked repository and that the command would also
+   display its own lock file which can be confusing.
+
+   Now, the `list locks` command never locks the repository.
+
+   https://github.com/restic/restic/issues/1106
+   https://github.com/restic/restic/pull/3665
+
+ * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes
+
+   The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of
+   restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a
+   temporary location before being moved to their proper location.
+
+   This also allows multiple concurrent restic processes to operate on a single repository
+   without conflicts. Previously, concurrent operations could cause segfaults because the
+   processes saw each other's partially downloaded files.
+
+   https://github.com/restic/restic/issues/2345
+   https://github.com/restic/restic/pull/2838
+
+ * Bugfix #2452: Improve error handling of repository locking
+
+   Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly
+   created one. Instead it continued trying to delete the old (usually no longer existing) lock
+   file and thus over time lots of lock files accumulated. This has now been fixed.
+
+   https://github.com/restic/restic/issues/2452
+   https://github.com/restic/restic/issues/2473
+   https://github.com/restic/restic/issues/2562
+   https://github.com/restic/restic/pull/3512
+
+ * Bugfix #2738: Don't print progress for `backup --json --quiet`
+
+   Unlike the text output, the `--json` output format still printed progress information even in
+   `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode.
+
+   https://github.com/restic/restic/issues/2738
+   https://github.com/restic/restic/pull/3264
+
+ * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+
+   Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable,
+   which caused problems in certain system/usage configurations. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3382
+   https://github.com/restic/restic/pull/3474
+
+ * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository
+
+   The `copy` command previously did not respect the `--no-lock` option for the source
+   repository, causing failures with read-only storage backends. This has now been fixed such
+   that the option is now respected.
+
+   https://github.com/restic/restic/issues/3518
+   https://github.com/restic/restic/pull/3589
+
+ * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
+
+   Previously, if a request failed with an SSL unknown certificate authority error, the B2
+   backend retried indefinitely and restic would appear to hang.
+
+   This has now been fixed and restic instead fails with an error message.
+
+   https://github.com/restic/restic/issues/3556
+   https://github.com/restic/restic/issues/2355
+   https://github.com/restic/restic/pull/3571
+
+ * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+
+   Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the
+   backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time
+   as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone
+   stdio connection already closed" errors.
+
+   This has now been fixed by restic starting the rclone process detached from the console restic
+   runs in (similar to starting processes in a new process group on Linux), which enables restic to
+   gracefully clean up rclone (which now never gets the `SIGINT`).
+
+   https://github.com/restic/restic/issues/3601
+   https://github.com/restic/restic/pull/3602
+
+ * Bugfix #3667: The `mount` command now reports symlinks sizes
+
+   Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They
+   now have a size equal to the byte length of their target path, as required by POSIX.
+
+   https://github.com/restic/restic/issues/3667
+   https://github.com/restic/restic/pull/3668
+
+ * Bugfix #3488: `rebuild-index` failed if an index file was damaged
+
+   Previously, the `rebuild-index` command would fail with an error if an index file was damaged
+   or truncated. This has now been fixed.
+
+   On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or
+   to manually delete the damaged index.
+
+   https://github.com/restic/restic/pull/3488
+
+ * Bugfix #3591: Fix handling of `prune --max-repack-size=0`
+
+   Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been
+   fixed.
+
+   As a workaround, `--max-repack-size=1` can be used with older versions of restic.
+
+   https://github.com/restic/restic/pull/3591
+
+ * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot
+
+   The `backup` command, when a `--parent` was not provided, previously chose the most recent
+   matching snapshot as the parent snapshot. However, this didn't make sense when the user passed
+   `--time` to create a new snapshot older than the most recent snapshot.
+
+   Instead, `backup` now chooses the most recent snapshot which is not newer than the
+   snapshot-being-created's timestamp, to avoid any time travel.
+
+   https://github.com/restic/restic/pull/3619
+
+ * Change #3641: Ignore parent snapshot for `backup --stdin`
+
+   Restic uses a parent snapshot to speed up directory scanning when performing backups, but this
+   only wasted time and memory when the backup source is stdin (using the `--stdin` option of the
+   `backup` command), since no directory scanning is performed in this case.
+
+   Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to
+   skip some startup operations and saves a bit of resources.
+
+   The `--parent` option is still available for `backup --stdin`, but is now ignored.
+
+   https://github.com/restic/restic/issues/3641
+   https://github.com/restic/restic/pull/3645
+
+ * Change #3519: Require Go 1.14 or newer
+
+   Restic now requires Go 1.14 to build. This allows it to use new standard library features
+   instead of an external dependency.
+
+   https://github.com/restic/restic/issues/3519
+
+ * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command
+
+   Testing exclude filters and other configuration options was error prone as wrong filters
+   could cause files to be uploaded unintentionally. It was also not possible to estimate
+   beforehand how much data would be uploaded.
+
+   The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of
+   a backup without actually writing anything to the repository.
+
+   Passing -vv will log information about files that would be added, allowing for verification of
+   source and exclusion options before running the real backup.
+
+   https://github.com/restic/restic/issues/1542
+   https://github.com/restic/restic/pull/2308
+   https://github.com/restic/restic/pull/3210
+   https://github.com/restic/restic/pull/3300
+
+ * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends
+
+   Previously only the B2 and partially the Swift backends verified the integrity of uploaded
+   (encrypted) files. The verification works by informing the backend about the expected hash of
+   the uploaded file. The backend then verifies the upload and thereby rules out any data
+   corruption during upload.
+
+   We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides
+   integrity checking for uploads also means that restic can now be used to store backups in S3
+   buckets which have Object Lock enabled.
+
+   https://github.com/restic/restic/issues/2202
+   https://github.com/restic/restic/issues/2700
+   https://github.com/restic/restic/issues/3023
+   https://github.com/restic/restic/pull/3246
+
+ * Enhancement #233: Support negative include/exclude patterns
+
+   If a pattern starts with an exclamation mark and it matches a file that was previously matched by
+   a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to
+   cancel the exclusion of some files.
+
+   It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it
+   is not possible to include files inside the directory.
+
+   Example of use as an exclude pattern for the `backup` command:
+
+   $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...]
+   node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...]
+
+   https://github.com/restic/restic/issues/233
+   https://github.com/restic/restic/pull/2311
+
+ * Enhancement #2388: Add warning for S3 if partial credentials are provided
+
+   Previously restic did not notify about incomplete credentials when using the S3 backend,
+   instead just reporting access denied.
+
+   Restic now checks that both the AWS key ID and secret environment variables are set before
+   connecting to the remote server, and reports an error if not.
+
+   https://github.com/restic/restic/issues/2388
+   https://github.com/restic/restic/pull/3532
+
+ * Enhancement #2508: Support JSON output and quiet mode for the `diff` command
+
+   The `diff` command now supports outputting machine-readable output in JSON format. To enable
+   this, pass the `--json` option to the command. To only print the summary and suppress detailed
+   output, pass the `--quiet` option.
+
+   https://github.com/restic/restic/issues/2508
+   https://github.com/restic/restic/pull/3592
+
+ * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates
+
+   There is now an `--insecure-tls` global option in restic, which disables TLS verification for
+   self-signed certificates in order to support some development workflows.
+
+   https://github.com/restic/restic/issues/2656
+   https://github.com/restic/restic/pull/2657
+
+ * Enhancement #3003: Atomic uploads for the SFTP backend
+
+   The SFTP backend did not upload files atomically. An interrupted upload could leave an
+   incomplete file behind which could prevent restic from accessing the repository. This has now
+   been fixed and uploads in the SFTP backend are done atomically.
+
+   https://github.com/restic/restic/issues/3003
+   https://github.com/restic/restic/pull/3524
+
+ * Enhancement #3127: Add xattr (extended attributes) support for Solaris
+
+   Restic now supports xattr for the Solaris operating system.
+
+   https://github.com/restic/restic/issues/3127
+   https://github.com/restic/restic/pull/3628
+
+ * Enhancement #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+
+   Restic used to silently ignore the `--no-lock` option of the `forget` command.
+
+   It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If
+   `--no-lock` option is specified without `--dry-run`, restic prints a warning message to
+   stderr.
+
+   https://github.com/restic/restic/issues/3464
+   https://github.com/restic/restic/pull/3623
+
+ * Enhancement #3490: Support random subset by size in `check --read-data-subset`
+
+   The `--read-data-subset` option of the `check` command now supports a third way of specifying
+   the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or
+   t/T.
+
+   https://github.com/restic/restic/issues/3490
+   https://github.com/restic/restic/pull/3548
+
+ * Enhancement #3541: Improve handling of temporary B2 delete errors
+
+   Deleting files on B2 could sometimes fail temporarily, which required restic to retry the
+   delete operation. In some cases the file was deleted nevertheless, causing the retries and
+   ultimately the restic command to fail. This has now been fixed.
+
+   https://github.com/restic/restic/issues/3541
+   https://github.com/restic/restic/pull/3544
+
+ * Enhancement #3542: Add file mode in symbolic notation to `ls --json`
+
+   The `ls --json` command now provides the file mode in symbolic notation (using the
+   `permissions` key), aligned with `find --json`.
+
+   https://github.com/restic/restic/issues/3542
+   https://github.com/restic/restic/pull/3573
+   https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
+
+ * Enhancement #2594: Speed up the `restore --verify` command
+
+   The `--verify` option lets the `restore` command verify the file content after it has restored
+   a snapshot. The performance of this operation has now been improved by up to a factor of two.
+
+   https://github.com/restic/restic/pull/2594
+
+ * Enhancement #2816: The `backup` command no longer updates file access times on Linux
+
+   When reading files during backup, restic used to cause the operating system to update the
+   files' access times. Note that this did not apply to filesystems with disabled file access
+   times.
+
+   Restic now instructs the operating system not to update the file access time, if the user
+   running restic is the file owner or has root permissions.
+
+   https://github.com/restic/restic/pull/2816
+
+ * Enhancement #2880: Make `recover` collect only unreferenced trees
+
+   Previously, the `recover` command used to generate a snapshot containing *all* root trees,
+   even those which were already referenced by a snapshot.
+
+   This has been improved such that it now only processes trees not already referenced by any
+   snapshot.
+
+   https://github.com/restic/restic/pull/2880
+
+ * Enhancement #3429: Verify that new or modified keys are stored correctly
+
+   When adding a new key or changing the password of a key, restic used to just create the new key (and
+   remove the old one, when changing the password). There was no verification that the new key was
+   stored correctly and works properly. As the repository cannot be decrypted without a valid key
+   file, this could in rare cases cause the repository to become inaccessible.
+
+   Restic now checks that new key files actually work before continuing. This can protect against
+   some (rare) cases of hardware or storage problems.
+
+   https://github.com/restic/restic/pull/3429
+
+ * Enhancement #3436: Improve local backend's resilience to (system) crashes
+
+   Restic now ensures that files stored using the `local` backend are created atomically (that
+   is, files are either stored completely or not at all). This ensures that no incomplete files are
+   left behind even if restic is terminated while writing a file.
+
+   In addition, restic now tries to ensure that the directory in the repository which contains a
+   newly uploaded file is also written to disk. This can prevent missing files if the system
+   crashes or the disk is not properly unmounted.
+
+   https://github.com/restic/restic/pull/3436
+
+ * Enhancement #3508: Cache blobs read by the `dump` command
+
+   When dumping a file using the `dump` command, restic did not cache blobs in any way, so even
+   consecutive runs of the same blob were loaded from the repository again and again, slowing down
+   the dump.
+
+   Now, the caching mechanism already used by the `fuse` command is also used by the `dump`
+   command. This makes dumping much faster, especially for sparse files.
+
+   https://github.com/restic/restic/pull/3508
+
+ * Enhancement #3511: Support configurable timeout for the rclone backend
+
+   A slow rclone backend could cause restic to time out while waiting for the repository to open.
+   Restic now offers an `-o rclone.timeout` option to make this timeout configurable.
+
+   https://github.com/restic/restic/issues/3511
+   https://github.com/restic/restic/pull/3514
+
+ * Enhancement #3593: Improve `copy` performance by parallelizing IO
+
+   Restic copy previously only used a single thread for copying blobs between repositories,
+   which resulted in limited performance when copying small blobs to/from a high latency backend
+   (i.e. any remote backend, especially b2).
+
+   Copying will now use 8 parallel threads to increase the throughput of the copy operation.
+
+   https://github.com/restic/restic/pull/3593
+
+
 Changelog for restic 0.12.1 (2021-08-03)
 =======================================
 
@@ -3083,7 +3524,7 @@ Summary
  * Enh #1055: Create subdirs below `data/` for local/sftp backends
  * Enh #1067: Allow loading credentials for s3 from IAM
  * Enh #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
- * Enh #1081: Clarify semantic for `--tasg` for the `forget` command
+ * Enh #1081: Clarify semantic for `--tag` for the `forget` command
  * Enh #1080: Ignore chmod() errors on filesystems which do not support it
  * Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t)
 
@@ -3127,7 +3568,7 @@ Details
    https://github.com/restic/restic/issues/1073
    https://github.com/restic/restic/pull/1075
 
- * Enhancement #1081: Clarify semantic for `--tasg` for the `forget` command
+ * Enhancement #1081: Clarify semantic for `--tag` for the `forget` command
 
    https://github.com/restic/restic/issues/1081
    https://github.com/restic/restic/pull/1090

CONTRIBUTING.md

@@ -66,7 +66,7 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.13 or later for compiling. Clone the repo (without
+Restic requires Go version 1.14 or later for compiling. Clone the repo (without
 having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH
@@ -123,7 +123,10 @@ down to the following steps:
     writing, ask yourself: If I were the user, what would I need to be aware
     of with this change?
 
- 8. Once your code looks good and passes all the tests, we'll merge it. Thanks
+ 8. Do not edit the man pages under `doc/man` or `doc/manual_rest.rst` -
+    these are autogenerated before new releases.
+
+ 9. Once your code looks good and passes all the tests, we'll merge it. Thanks
     a lot for your contribution!
 
 Please provide the patches for each bug or feature in a separate branch and

README.md

@@ -46,8 +46,8 @@ Therefore, restic supports the following backends for storing backups natively:
 
 - [Local directory](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local)
 - [sftp server (via SSH)](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp)
-- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](doc/100_references.rst#rest-backend), [rest-server](https://github.com/restic/rest-server))
-- [AWS S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
+- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](https://restic.readthedocs.io/en/latest/100_references.html#rest-backend), [rest-server](https://github.com/restic/rest-server))
+- [Amazon S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
 - [OpenStack Swift](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift)
 - [BackBlaze B2](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2)
 - [Microsoft Azure Blob Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)

VERSION

@@ -1 +1 @@
-0.12.1
+0.13.1

build.go

@@ -35,6 +35,7 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+//go:build ignore_build_go
 // +build ignore_build_go
 
 package main
@@ -58,7 +59,7 @@ var config = Config{
 	Main:             "./cmd/restic",                           // package name for the main package
 	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
 	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 11, Patch: 0}, // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
@@ -123,17 +124,8 @@ func printEnv(env []string) {
 
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd string, env map[string]string, args ...string) error {
-	a := []string{"build"}
-
-	// try to remove all absolute paths from resulting binary
-	if goVersion.AtLeast(GoVersion{1, 13, 0}) {
-		// use the new flag introduced by Go 1.13
-		a = append(a, "-trimpath")
-	} else {
-		// otherwise try to trim as many paths as possible
-		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
-	}
+	// -trimpath removes all absolute paths from the binary.
+	a := []string{"build", "-trimpath"}
 
 	if enablePIE {
 		a = append(a, "-buildmode=pie")

changelog/0.13.0_2022-03-26/issue-1106

@@ -0,0 +1,10 @@
+Bugfix: Never lock repository for `list locks`
+
+The `list locks` command previously locked to the repository by default. This
+had the problem that it wouldn't work for an exclusively locked repository and
+that the command would also display its own lock file which can be confusing.
+
+Now, the `list locks` command never locks the repository.
+
+https://github.com/restic/restic/issues/1106
+https://github.com/restic/restic/pull/3665

changelog/0.13.0_2022-03-26/issue-1542

@@ -0,0 +1,16 @@
+Enhancement: Add `--dry-run`/`-n` option to `backup` command
+
+Testing exclude filters and other configuration options was error prone as
+wrong filters could cause files to be uploaded unintentionally. It was also
+not possible to estimate beforehand how much data would be uploaded.
+
+The `backup` command now has a `--dry-run`/`-n` option, which performs all the
+normal steps of a backup without actually writing anything to the repository.
+
+Passing -vv will log information about files that would be added, allowing for
+verification of source and exclusion options before running the real backup.
+
+https://github.com/restic/restic/issues/1542
+https://github.com/restic/restic/pull/2308
+https://github.com/restic/restic/pull/3210
+https://github.com/restic/restic/pull/3300

changelog/0.13.0_2022-03-26/issue-2202

@@ -0,0 +1,15 @@
+Enhancement: Add upload checksum for Azure, GS, S3 and Swift backends
+
+Previously only the B2 and partially the Swift backends verified the integrity
+of uploaded (encrypted) files. The verification works by informing the backend
+about the expected hash of the uploaded file. The backend then verifies the
+upload and thereby rules out any data corruption during upload.
+
+We have now added upload checksums for the Azure, GS, S3 and Swift backends,
+which besides integrity checking for uploads also means that restic can now be
+used to store backups in S3 buckets which have Object Lock enabled.
+
+https://github.com/restic/restic/issues/2202
+https://github.com/restic/restic/issues/2700
+https://github.com/restic/restic/issues/3023
+https://github.com/restic/restic/pull/3246

changelog/0.13.0_2022-03-26/issue-233

@@ -0,0 +1,29 @@
+Enhancement: Support negative include/exclude patterns
+
+If a pattern starts with an exclamation mark and it matches a file that was
+previously matched by a regular pattern, the match is cancelled. Notably,
+this can be used with `--exclude-file` to cancel the exclusion of some files.
+
+It works similarly to `.gitignore`, with the same limitation; Once a directory
+is excluded, it is not possible to include files inside the directory.
+
+Example of use as an exclude pattern for the `backup` command:
+
+    $HOME/**/*
+    !$HOME/Documents
+    !$HOME/code
+    !$HOME/.emacs.d
+    !$HOME/games
+    # [...]
+    node_modules
+    *~
+    *.o
+    *.lo
+    *.pyc
+    # [...]
+    $HOME/code/linux/*
+    !$HOME/code/linux/.git
+    # [...]
+
+https://github.com/restic/restic/issues/233
+https://github.com/restic/restic/pull/2311

changelog/0.13.0_2022-03-26/issue-2345

@@ -0,0 +1,13 @@
+Bugfix: Make cache crash-resistant and usable by multiple concurrent processes
+
+The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state
+in the event of restic (or the OS) crashing. This is now less likely to occur
+as files are downloaded to a temporary location before being moved to their
+proper location.
+
+This also allows multiple concurrent restic processes to operate on a single
+repository without conflicts. Previously, concurrent operations could cause
+segfaults because the processes saw each other's partially downloaded files.
+
+https://github.com/restic/restic/issues/2345
+https://github.com/restic/restic/pull/2838

changelog/0.13.0_2022-03-26/issue-2388

@@ -0,0 +1,10 @@
+Enhancement: Add warning for S3 if partial credentials are provided
+
+Previously restic did not notify about incomplete credentials when using the
+S3 backend, instead just reporting access denied.
+
+Restic now checks that both the AWS key ID and secret environment variables are
+set before connecting to the remote server, and reports an error if not.
+
+https://github.com/restic/restic/issues/2388
+https://github.com/restic/restic/pull/3532

changelog/0.13.0_2022-03-26/issue-2452

@@ -0,0 +1,11 @@
+Bugfix: Improve error handling of repository locking
+
+Previously, when the lock refresh failed to delete the old lock file, it forgot
+about the newly created one. Instead it continued trying to delete the old
+(usually no longer existing) lock file and thus over time lots of lock files
+accumulated. This has now been fixed.
+
+https://github.com/restic/restic/issues/2452
+https://github.com/restic/restic/issues/2473
+https://github.com/restic/restic/issues/2562
+https://github.com/restic/restic/pull/3512

changelog/0.13.0_2022-03-26/issue-2508

@@ -0,0 +1,8 @@
+Enhancement: Support JSON output and quiet mode for the `diff` command
+
+The `diff` command now supports outputting machine-readable output in JSON
+format. To enable this, pass the `--json` option to the command. To only print
+the summary and suppress detailed output, pass the `--quiet` option.
+
+https://github.com/restic/restic/issues/2508
+https://github.com/restic/restic/pull/3592

changelog/0.13.0_2022-03-26/issue-2656

@@ -0,0 +1,8 @@
+Enhancement: Add flag to disable TLS verification for self-signed certificates
+
+There is now an `--insecure-tls` global option in restic, which disables TLS
+verification for self-signed certificates in order to support some development
+workflows.
+
+https://github.com/restic/restic/issues/2656
+https://github.com/restic/restic/pull/2657

changelog/0.13.0_2022-03-26/issue-2738

@@ -0,0 +1,8 @@
+Bugfix: Don't print progress for `backup --json --quiet`
+
+Unlike the text output, the `--json` output format still printed progress
+information even in `--quiet` mode. This has now been fixed by always
+disabling the progress output in quiet mode.
+
+https://github.com/restic/restic/issues/2738
+https://github.com/restic/restic/pull/3264

changelog/0.13.0_2022-03-26/issue-3003

@@ -0,0 +1,9 @@
+Enhancement: Atomic uploads for the SFTP backend
+
+The SFTP backend did not upload files atomically. An interrupted upload could
+leave an incomplete file behind which could prevent restic from accessing the
+repository. This has now been fixed and uploads in the SFTP backend are done
+atomically.
+
+https://github.com/restic/restic/issues/3003
+https://github.com/restic/restic/pull/3524

changelog/0.13.0_2022-03-26/issue-3127

@@ -0,0 +1,6 @@
+Enhancement: Add xattr (extended attributes) support for Solaris
+
+Restic now supports xattr for the Solaris operating system.
+
+https://github.com/restic/restic/issues/3127
+https://github.com/restic/restic/pull/3628

changelog/0.13.0_2022-03-26/issue-3382

@@ -0,0 +1,8 @@
+Bugfix: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
+
+Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment
+variable, which caused problems in certain system/usage configurations. This
+has now been fixed.
+
+https://github.com/restic/restic/issues/3382
+https://github.com/restic/restic/pull/3474

changelog/0.13.0_2022-03-26/issue-3464

@@ -0,0 +1,10 @@
+Enhancement: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+
+Restic used to silently ignore the `--no-lock` option of the `forget` command.
+
+It now skips creation of lock file in case both `--dry-run` and `--no-lock`
+are specified. If `--no-lock` option is specified without `--dry-run`, restic
+prints a warning message to stderr.
+
+https://github.com/restic/restic/issues/3464
+https://github.com/restic/restic/pull/3623

changelog/0.13.0_2022-03-26/issue-3490

@@ -0,0 +1,8 @@
+Enhancement: Support random subset by size in `check --read-data-subset`
+
+The `--read-data-subset` option of the `check` command now supports a third way
+of specifying the subset to check, namely `nS` where `n` is a size in bytes with
+suffix `S` as k/K, m/M, g/G or t/T.
+
+https://github.com/restic/restic/issues/3490
+https://github.com/restic/restic/pull/3548

changelog/0.13.0_2022-03-26/issue-3518

@@ -0,0 +1,8 @@
+Bugfix: Make `copy` command honor `--no-lock` for source repository
+
+The `copy` command previously did not respect the `--no-lock` option for the
+source repository, causing failures with read-only storage backends. This has
+now been fixed such that the option is now respected.
+
+https://github.com/restic/restic/issues/3518
+https://github.com/restic/restic/pull/3589

changelog/0.13.0_2022-03-26/issue-3541

@@ -0,0 +1,9 @@
+Enhancement: Improve handling of temporary B2 delete errors
+
+Deleting files on B2 could sometimes fail temporarily, which required restic to
+retry the delete operation. In some cases the file was deleted nevertheless,
+causing the retries and ultimately the restic command to fail. This has now been
+fixed.
+
+https://github.com/restic/restic/issues/3541
+https://github.com/restic/restic/pull/3544

changelog/0.13.0_2022-03-26/issue-3542

@@ -0,0 +1,8 @@
+Enhancement: Add file mode in symbolic notation to `ls --json`
+
+The `ls --json` command now provides the file mode in symbolic notation (using
+the `permissions` key), aligned with `find --json`.
+
+https://github.com/restic/restic/issues/3542
+https://github.com/restic/restic/pull/3573
+https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371

changelog/0.13.0_2022-03-26/issue-3556

@@ -0,0 +1,10 @@
+Bugfix: Fix hang with Backblaze B2 on SSL certificate authority error
+
+Previously, if a request failed with an SSL unknown certificate authority
+error, the B2 backend retried indefinitely and restic would appear to hang.
+
+This has now been fixed and restic instead fails with an error message.
+
+https://github.com/restic/restic/issues/3556
+https://github.com/restic/restic/issues/2355
+https://github.com/restic/restic/pull/3571

changelog/0.13.0_2022-03-26/issue-3601

@@ -0,0 +1,15 @@
+Bugfix: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
+
+Previously, pressing Ctrl+C in a Windows console where restic was running with
+rclone as the backend would cause rclone to exit prematurely due to getting a
+`SIGINT` signal at the same time as restic. Restic would then wait for a long
+time for time with "unexpected EOF" and "rclone stdio connection already closed"
+errors.
+
+This has now been fixed by restic starting the rclone process detached from the
+console restic runs in (similar to starting processes in a new process group on
+Linux), which enables restic to gracefully clean up rclone (which now never gets
+the `SIGINT`).
+
+https://github.com/restic/restic/issues/3601
+https://github.com/restic/restic/pull/3602

changelog/0.13.0_2022-03-26/issue-3641

@@ -0,0 +1,14 @@
+Change: Ignore parent snapshot for `backup --stdin`
+
+Restic uses a parent snapshot to speed up directory scanning when performing
+backups, but this only wasted time and memory when the backup source is stdin
+(using the `--stdin` option of the `backup` command), since no directory scanning
+is performed in this case.
+
+Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows
+restic to skip some startup operations and saves a bit of resources.
+
+The `--parent` option is still available for `backup --stdin`, but is now ignored.
+
+https://github.com/restic/restic/issues/3641
+https://github.com/restic/restic/pull/3645

changelog/0.13.0_2022-03-26/issue-3667

@@ -0,0 +1,8 @@
+Bugfix: The `mount` command now reports symlinks sizes
+
+Symlinks used to have size zero in restic mountpoints, confusing some
+third-party tools. They now have a size equal to the byte length of their
+target path, as required by POSIX.
+
+https://github.com/restic/restic/issues/3667
+https://github.com/restic/restic/pull/3668

changelog/0.13.0_2022-03-26/pull-2594

@@ -0,0 +1,7 @@
+Enhancement: Speed up the `restore --verify` command
+
+The `--verify` option lets the `restore` command verify the file content
+after it has restored a snapshot. The performance of this operation has
+now been improved by up to a factor of two.
+
+https://github.com/restic/restic/pull/2594

changelog/0.13.0_2022-03-26/pull-2816

@@ -0,0 +1,10 @@
+Enhancement: The `backup` command no longer updates file access times on Linux
+
+When reading files during backup, restic used to cause the operating system to
+update the files' access times. Note that this did not apply to filesystems with
+disabled file access times.
+
+Restic now instructs the operating system not to update the file access time,
+if the user running restic is the file owner or has root permissions.
+
+https://github.com/restic/restic/pull/2816

changelog/0.13.0_2022-03-26/pull-2880

@@ -0,0 +1,9 @@
+Enhancement: Make `recover` collect only unreferenced trees
+
+Previously, the `recover` command used to generate a snapshot containing *all*
+root trees, even those which were already referenced by a snapshot.
+
+This has been improved such that it now only processes trees not already
+referenced by any snapshot.
+
+https://github.com/restic/restic/pull/2880

changelog/0.13.0_2022-03-26/pull-3429

@@ -0,0 +1,12 @@
+Enhancement: Verify that new or modified keys are stored correctly
+
+When adding a new key or changing the password of a key, restic used to just
+create the new key (and remove the old one, when changing the password). There
+was no verification that the new key was stored correctly and works properly.
+As the repository cannot be decrypted without a valid key file, this could in
+rare cases cause the repository to become inaccessible.
+
+Restic now checks that new key files actually work before continuing. This
+can protect against some (rare) cases of hardware or storage problems.
+
+https://github.com/restic/restic/pull/3429

changelog/0.13.0_2022-03-26/pull-3436

@@ -0,0 +1,12 @@
+Enhancement: Improve local backend's resilience to (system) crashes
+
+Restic now ensures that files stored using the `local` backend are created
+atomically (that is, files are either stored completely or not at all). This
+ensures that no incomplete files are left behind even if restic is terminated
+while writing a file.
+
+In addition, restic now tries to ensure that the directory in the repository
+which contains a newly uploaded file is also written to disk. This can prevent
+missing files if the system crashes or the disk is not properly unmounted.
+
+https://github.com/restic/restic/pull/3436

changelog/0.13.0_2022-03-26/pull-3488

@@ -0,0 +1,9 @@
+Bugfix: `rebuild-index` failed if an index file was damaged
+
+Previously, the `rebuild-index` command would fail with an error if an index
+file was damaged or truncated. This has now been fixed.
+
+On older restic versions, a (slow) workaround is to use
+`rebuild-index --read-all-packs` or to manually delete the damaged index.
+
+https://github.com/restic/restic/pull/3488

changelog/0.13.0_2022-03-26/pull-3508

@@ -0,0 +1,10 @@
+Enhancement: Cache blobs read by the `dump` command
+
+When dumping a file using the `dump` command, restic did not cache blobs in any
+way, so even consecutive runs of the same blob were loaded from the repository
+again and again, slowing down the dump.
+
+Now, the caching mechanism already used by the `fuse` command is also used by
+the `dump` command. This makes dumping much faster, especially for sparse files.
+
+https://github.com/restic/restic/pull/3508

changelog/0.13.0_2022-03-26/pull-3514

@@ -0,0 +1,8 @@
+Enhancement: Support configurable timeout for the rclone backend
+
+A slow rclone backend could cause restic to time out while waiting for the
+repository to open. Restic now offers an `-o rclone.timeout` option to make
+this timeout configurable.
+
+https://github.com/restic/restic/issues/3511
+https://github.com/restic/restic/pull/3514

changelog/0.13.0_2022-03-26/pull-3519

@@ -0,0 +1,6 @@
+Change: Require Go 1.14 or newer
+
+Restic now requires Go 1.14 to build. This allows it to use new
+standard library features instead of an external dependency.
+
+https://github.com/restic/restic/issues/3519

changelog/0.13.0_2022-03-26/pull-3591

@@ -0,0 +1,8 @@
+Bugfix: Fix handling of `prune --max-repack-size=0`
+
+Restic ignored the `--max-repack-size` option when passing a value of 0. This
+has now been fixed.
+
+As a workaround, `--max-repack-size=1` can be used with older versions of restic.
+
+https://github.com/restic/restic/pull/3591

changelog/0.13.0_2022-03-26/pull-3593

@@ -0,0 +1,10 @@
+Enhancement: Improve `copy` performance by parallelizing IO
+
+Restic copy previously only used a single thread for copying blobs between
+repositories, which resulted in limited performance when copying small blobs
+to/from a high latency backend (i.e. any remote backend, especially b2).
+
+Copying will now use 8 parallel threads to increase the throughput of the copy
+operation.
+
+https://github.com/restic/restic/pull/3593

changelog/0.13.0_2022-03-26/pull-3619

@@ -0,0 +1,11 @@
+Bugfix: Avoid choosing parent snapshots newer than time of new snapshot
+
+The `backup` command, when a `--parent` was not provided, previously chose the
+most recent matching snapshot as the parent snapshot. However, this didn't make
+sense when the user passed `--time` to create a new snapshot older than the most
+recent snapshot.
+
+Instead, `backup` now chooses the most recent snapshot which is not newer than
+the snapshot-being-created's timestamp, to avoid any time travel.
+
+https://github.com/restic/restic/pull/3619

changelog/0.13.1_2022-04-10/issue-3685

@@ -0,0 +1,7 @@
+Bugfix: Fix the diff command
+
+There was a bug in the `diff` command, it would always show files in a removed
+directory as added. We've fixed that.
+
+https://github.com/restic/restic/issues/3685
+https://github.com/restic/restic/pull/3686

changelog/0.13.1_2022-04-10/issue-3692

@@ -0,0 +1,12 @@
+Bugfix: Fix rclone (shimmed by Scoop) and sftp stopped working on Windows
+
+In #3602 a fix was introduced to fix the problem that rclone prematurely exits
+when Ctrl+C is pressed on Windows. The solution was to create the subprocess
+with its console detached from the restic console. However, such solution
+fails when using rclone install by scoop or using sftp with a passphrase-
+protected private key. We've fixed that by using a different approach to prevent
+Ctrl-C from passing down too early.
+
+https://github.com/restic/restic/issues/3681
+https://github.com/restic/restic/issues/3692
+https://github.com/restic/restic/pull/3696

changelog/0.7.1_2017-07-22/issue-1081

@@ -1,4 +1,4 @@
-Enhancement: Clarify semantic for `--tasg` for the `forget` command
+Enhancement: Clarify semantic for `--tag` for the `forget` command
 
 https://github.com/restic/restic/issues/1081
 https://github.com/restic/restic/pull/1090

cmd/restic/cleanup.go

@@ -58,7 +58,7 @@ func RunCleanupHandlers() {
 func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
-		Warnf("%ssignal %v received, cleaning up\n", ClearLine(), s)
+		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
 
 		code := 0
 

cmd/restic/cmd_backup.go

@@ -24,8 +24,7 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui"
-	"github.com/restic/restic/internal/ui/json"
+	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
@@ -92,6 +91,7 @@ type BackupOptions struct {
 	IgnoreInode             bool
 	IgnoreCtime             bool
 	UseFsSnapshot           bool
+	DryRun                  bool
 }
 
 var backupOptions BackupOptions
@@ -103,7 +103,7 @@ func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
@@ -132,6 +132,7 @@ func init() {
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
@@ -471,7 +472,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
 		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
@@ -484,7 +485,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host})
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -525,33 +526,17 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	type ArchiveProgressReporter interface {
-		CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration)
-		StartFile(filename string)
-		CompleteBlob(filename string, bytes uint64)
-		ScannerError(item string, fi os.FileInfo, err error) error
-		ReportTotal(item string, s archiver.ScanStats)
-		SetMinUpdatePause(d time.Duration)
-		Run(ctx context.Context) error
-		Error(item string, fi os.FileInfo, err error) error
-		Finish(snapshotID restic.ID)
-
-		// ui.StdioWrapper
-		Stdout() io.WriteCloser
-		Stderr() io.WriteCloser
-
-		// ui.Message
-		E(msg string, args ...interface{})
-		P(msg string, args ...interface{})
-		V(msg string, args ...interface{})
-		VV(msg string, args ...interface{})
-	}
-
-	var p ArchiveProgressReporter
+	var progressPrinter backup.ProgressPrinter
 	if gopts.JSON {
-		p = json.NewBackup(term, gopts.verbosity)
+		progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
 	} else {
-		p = ui.NewBackup(term, gopts.verbosity)
+		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
+	}
+	progressReporter := backup.NewProgress(progressPrinter)
+
+	if opts.DryRun {
+		repo.SetDryRun()
+		progressReporter.SetDryRun()
 	}
 
 	// use the terminal for stdout/stderr
@@ -559,14 +544,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	defer func() {
 		gopts.stdout, gopts.stderr = prevStdout, prevStderr
 	}()
-	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
+	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
 
-	p.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet))
+	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 
-	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
+	t.Go(func() error { return progressReporter.Run(t.Context(gopts.ctx)) })
 
 	if !gopts.JSON {
-		p.V("lock repository")
+		progressPrinter.V("lock repository")
 	}
 	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
@@ -587,23 +572,26 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	if !gopts.JSON {
-		p.V("load index files")
+		progressPrinter.V("load index files")
 	}
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 
-	parentSnapshotID, err := findParentSnapshot(gopts.ctx, repo, opts, targets)
-	if err != nil {
-		return err
-	}
+	var parentSnapshotID *restic.ID
+	if !opts.Stdin {
+		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
+		if err != nil {
+			return err
+		}
 
-	if !gopts.JSON {
-		if parentSnapshotID != nil {
-			p.P("using parent snapshot %v\n", parentSnapshotID.Str())
-		} else {
-			p.P("no parent snapshot found, will read all files\n")
+		if !gopts.JSON {
+			if parentSnapshotID != nil {
+				progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
+			} else {
+				progressPrinter.P("no parent snapshot found, will read all files\n")
+			}
 		}
 	}
 
@@ -632,12 +620,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 
 		errorHandler := func(item string, err error) error {
-			return p.Error(item, nil, err)
+			return progressReporter.Error(item, nil, err)
 		}
 
 		messageHandler := func(msg string, args ...interface{}) {
 			if !gopts.JSON {
-				p.P(msg, args...)
+				progressPrinter.P(msg, args...)
 			}
 		}
 
@@ -647,7 +635,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 	if opts.Stdin {
 		if !gopts.JSON {
-			p.V("read data from stdin")
+			progressPrinter.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
 		targetFS = &fs.Reader{
@@ -662,11 +650,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	sc := archiver.NewScanner(targetFS)
 	sc.SelectByName = selectByNameFilter
 	sc.Select = selectFilter
-	sc.Error = p.ScannerError
-	sc.Result = p.ReportTotal
+	sc.Error = progressReporter.ScannerError
+	sc.Result = progressReporter.ReportTotal
 
 	if !gopts.JSON {
-		p.V("start scan on %v", targets)
+		progressPrinter.V("start scan on %v", targets)
 	}
 	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
 
@@ -677,11 +665,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	success := true
 	arch.Error = func(item string, fi os.FileInfo, err error) error {
 		success = false
-		return p.Error(item, fi, err)
+		return progressReporter.Error(item, fi, err)
 	}
-	arch.CompleteItem = p.CompleteItem
-	arch.StartFile = p.StartFile
-	arch.CompleteBlob = p.CompleteBlob
+	arch.CompleteItem = progressReporter.CompleteItem
+	arch.StartFile = progressReporter.StartFile
+	arch.CompleteBlob = progressReporter.CompleteBlob
 
 	if opts.IgnoreInode {
 		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
@@ -705,7 +693,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	if !gopts.JSON {
-		p.V("start backup on %v", targets)
+		progressPrinter.V("start backup on %v", targets)
 	}
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
 
@@ -721,9 +709,9 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	// Report finished execution
-	p.Finish(id)
-	if !gopts.JSON {
-		p.P("snapshot %s saved\n", id.Str())
+	progressReporter.Finish(id)
+	if !gopts.JSON && !opts.DryRun {
+		progressPrinter.P("snapshot %s saved\n", id.Str())
 	}
 	if !success {
 		return ErrInvalidSourceData

cmd/restic/cmd_cache.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/cache"
@@ -140,8 +141,13 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
 		}
 
+		name := entry.Name()
+		if !strings.HasPrefix(name, "restic-check-cache-") {
+			name = name[:10]
+		}
+
 		tab.AddRow(data{
-			entry.Name()[:10],
+			name,
 			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
 			old,
 			size,

cmd/restic/cmd_check.go

@@ -5,10 +5,12 @@ import (
 	"math/rand"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
@@ -54,7 +56,7 @@ func init() {
 
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
 	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
@@ -65,7 +67,7 @@ func checkFlags(opts CheckOptions) error {
 	}
 	if opts.ReadDataSubset != "" {
 		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
+		argumentError := errors.Fatal("check flag --read-data-subset has invalid value, please see documentation")
 		if err == nil {
 			if len(dataSubset) != 2 {
 				return argumentError
@@ -76,7 +78,7 @@ func checkFlags(opts CheckOptions) error {
 			if dataSubset[1] > totalBucketsMax {
 				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
 			}
-		} else {
+		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
 			percentage, err := parsePercentage(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
@@ -84,8 +86,19 @@ func checkFlags(opts CheckOptions) error {
 
 			if percentage <= 0.0 || percentage > 100.0 {
 				return errors.Fatal(
-					"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
+					"check flag --read-data-subset=x% x must be above 0.0% and at most 100.0%")
 			}
+
+		} else {
+			fileSize, err := parseSizeStr(opts.ReadDataSubset)
+			if err != nil {
+				return argumentError
+			}
+			if fileSize <= 0.0 {
+				return errors.Fatal(
+					"check flag --read-data-subset=n n must be above 0")
+			}
+
 		}
 	}
 
@@ -146,6 +159,9 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	}
 
 	cachedir := gopts.CacheDir
+	if cachedir == "" {
+		cachedir = cache.EnvDir()
+	}
 
 	// use a cache in a temporary directory
 	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
@@ -241,7 +257,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
+	var wg sync.WaitGroup
+
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
 		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
 		chkr.Structure(gopts.ctx, bar, errChan)
@@ -259,6 +279,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	// Wait for the progress bar to be complete before printing more below.
+	// Must happen after `errChan` is read from in the above loop to avoid
+	// deadlocking in the case of errors.
+	wg.Wait()
+
 	if opts.CheckUnused {
 		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
 			Verbosef("unused blob %v\n", id)
@@ -294,10 +319,27 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
 			packCount := uint64(len(packs))
 			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
+		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
+			percentage, err := parsePercentage(opts.ReadDataSubset)
+			if err == nil {
+				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
+				Verbosef("read %.1f%% of data packs\n", percentage)
+			}
 		} else {
-			percentage, _ := parsePercentage(opts.ReadDataSubset)
-			packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
-			Verbosef("read %.1f%% of data packs\n", percentage)
+			repoSize := int64(0)
+			allPacks := chkr.GetPacks()
+			for _, size := range allPacks {
+				repoSize += size
+			}
+			if repoSize == 0 {
+				return errors.Fatal("Cannot read from a repository having size 0")
+			}
+			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
+			if subsetSize > repoSize {
+				subsetSize = repoSize
+			}
+			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
+			Verbosef("read %d bytes of data packs\n", subsetSize)
 		}
 		if packs == nil {
 			return errors.Fatal("internal error: failed to select packs to check")
@@ -349,6 +391,11 @@ func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage floa
 		id := keys[idx[i]]
 		packs[id] = allPacks[id]
 	}
-
+	return packs
+}
+
+func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, repoSize int64) map[restic.ID]int64 {
+	subsetPercentage := (float64(subsetSize) / float64(repoSize)) * 100.0
+	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }

cmd/restic/cmd_check_test.go

@@ -129,3 +129,37 @@ func TestSelectNoRandomPacksByPercentage(t *testing.T) {
 	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
 }
+
+func TestSelectRandomPacksByFileSize(t *testing.T) {
+	var testPacks = make(map[restic.ID]int64)
+	for i := 1; i <= 10; i++ {
+		id := restic.NewRandomID()
+		// ensure unique ids
+		id[0] = byte(i)
+		testPacks[id] = 0
+	}
+
+	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
+	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
+
+	selectedPacks = selectRandomPacksByFileSize(testPacks, 10240, 51200)
+	rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected selection")
+	}
+
+	selectedPacks = selectRandomPacksByFileSize(testPacks, 500, 500)
+	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected item in selection")
+	}
+}
+
+func TestSelectNoRandomPacksByFileSize(t *testing.T) {
+	// that the a repository without pack files works
+	var testPacks = make(map[restic.ID]int64)
+	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
+	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
+}

cmd/restic/cmd_copy.go

@@ -73,10 +73,12 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	srcLock, err := lockRepo(ctx, srcRepo)
-	defer unlockRepo(srcLock)
-	if err != nil {
-		return err
+	if !gopts.NoLock {
+		srcLock, err := lockRepo(ctx, srcRepo)
+		defer unlockRepo(srcLock)
+		if err != nil {
+			return err
+		}
 	}
 
 	dstLock, err := lockRepo(ctx, dstRepo)
@@ -174,9 +176,12 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 
+const numCopyWorkers = 8
+
 func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
 	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
 
+	idChan := make(chan restic.ID)
 	wg, ctx := errgroup.WithContext(ctx)
 
 	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
@@ -186,9 +191,9 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	}, nil)
 
 	wg.Go(func() error {
+		defer close(idChan)
 		// reused buffer
 		var buf []byte
-
 		for tree := range treeStream {
 			if tree.Error != nil {
 				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
@@ -209,32 +214,44 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 				}
 			}
 
-			// TODO: parallelize blob down/upload
-
 			for _, entry := range tree.Nodes {
 				// Recursion into directories is handled by StreamTrees
 				// Copy the blobs for this file.
 				for _, blobID := range entry.Content {
-					// Do we already have this data blob?
-					if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
-						continue
-					}
-					debug.Log("Copying blob %s\n", blobID.Str())
-					var err error
-					buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
-					if err != nil {
-						return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
-					}
-
-					_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
-					if err != nil {
-						return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
+					select {
+					case idChan <- blobID:
+					case <-ctx.Done():
+						return ctx.Err()
 					}
 				}
 			}
-
 		}
 		return nil
 	})
+
+	for i := 0; i < numCopyWorkers; i++ {
+		wg.Go(func() error {
+			// reused buffer
+			var buf []byte
+			for blobID := range idChan {
+				// Do we already have this data blob?
+				if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
+					continue
+				}
+				debug.Log("Copying blob %s\n", blobID.Str())
+				var err error
+				buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
+				if err != nil {
+					return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
+				}
+
+				_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
+				if err != nil {
+					return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
+				}
+			}
+			return nil
+		})
+	}
 	return wg.Wait()
 }

cmd/restic/cmd_diff.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"path"
 	"reflect"
 	"sort"
@@ -62,15 +63,29 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string)
 
 // Comparer collects all things needed to compare two snapshots.
 type Comparer struct {
-	repo restic.Repository
-	opts DiffOptions
+	repo        restic.Repository
+	opts        DiffOptions
+	printChange func(change *Change)
+}
+
+type Change struct {
+	MessageType string `json:"message_type"` // "change"
+	Path        string `json:"path"`
+	Modifier    string `json:"modifier"`
+}
+
+func NewChange(path string, mode string) *Change {
+	return &Change{MessageType: "change", Path: path, Modifier: mode}
 }
 
 // DiffStat collects stats for all types of items.
 type DiffStat struct {
-	Files, Dirs, Others  int
-	DataBlobs, TreeBlobs int
-	Bytes                uint64
+	Files     int    `json:"files"`
+	Dirs      int    `json:"dirs"`
+	Others    int    `json:"others"`
+	DataBlobs int    `json:"data_blobs"`
+	TreeBlobs int    `json:"tree_blobs"`
+	Bytes     uint64 `json:"bytes"`
 }
 
 // Add adds stats information for node to s.
@@ -113,21 +128,14 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 	}
 }
 
-// DiffStats collects the differences between two snapshots.
-type DiffStats struct {
-	ChangedFiles                         int
-	Added                                DiffStat
-	Removed                              DiffStat
-	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
-}
-
-// NewDiffStats creates new stats for a diff run.
-func NewDiffStats() *DiffStats {
-	return &DiffStats{
-		BlobsBefore: restic.NewBlobSet(),
-		BlobsAfter:  restic.NewBlobSet(),
-		BlobsCommon: restic.NewBlobSet(),
-	}
+type DiffStatsContainer struct {
+	MessageType                          string         `json:"message_type"` // "statistics"
+	SourceSnapshot                       string         `json:"source_snapshot"`
+	TargetSnapshot                       string         `json:"target_snapshot"`
+	ChangedFiles                         int            `json:"changed_files"`
+	Added                                DiffStat       `json:"added"`
+	Removed                              DiffStat       `json:"removed"`
+	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
 }
 
 // updateBlobs updates the blob counters in the stats struct.
@@ -162,7 +170,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		if node.Type == "dir" {
 			name += "/"
 		}
-		Printf("%-5s%v\n", mode, name)
+		c.printChange(NewChange(name, mode))
 		stats.Add(node)
 		addBlobs(blobs, node)
 
@@ -221,7 +229,7 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
 	return tree1Nodes, tree2Nodes, uniqueNames
 }
 
-func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
+func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
 	tree1, err := c.repo.LoadTree(ctx, id1)
 	if err != nil {
@@ -265,7 +273,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			}
 
 			if mod != "" {
-				Printf("%-5s%v\n", mod, name)
+				c.printChange(NewChange(name, mod))
 			}
 
 			if node1.Type == "dir" && node2.Type == "dir" {
@@ -284,7 +292,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node1.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "-", prefix)
+			c.printChange(NewChange(prefix, "-"))
 			stats.Removed.Add(node1)
 
 			if node1.Type == "dir" {
@@ -298,7 +306,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node2.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "+", prefix)
+			c.printChange(NewChange(prefix, "+"))
 			stats.Added.Add(node2)
 
 			if node2.Type == "dir" {
@@ -348,7 +356,9 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	if !gopts.JSON {
+		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	}
 
 	if sn1.Tree == nil {
 		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
@@ -361,9 +371,33 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,
+		printChange: func(change *Change) {
+			Printf("%-5s%v\n", change.Modifier, change.Path)
+		},
 	}
 
-	stats := NewDiffStats()
+	if gopts.JSON {
+		enc := json.NewEncoder(gopts.stdout)
+		c.printChange = func(change *Change) {
+			err := enc.Encode(change)
+			if err != nil {
+				Warnf("JSON encode failed: %v\n", err)
+			}
+		}
+	}
+
+	if gopts.Quiet {
+		c.printChange = func(change *Change) {}
+	}
+
+	stats := &DiffStatsContainer{
+		MessageType:    "statistics",
+		SourceSnapshot: args[0],
+		TargetSnapshot: args[1],
+		BlobsBefore:    restic.NewBlobSet(),
+		BlobsAfter:     restic.NewBlobSet(),
+		BlobsCommon:    restic.NewBlobSet(),
+	}
 	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
 	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
 
@@ -376,14 +410,21 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
 	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
 
-	Printf("\n")
-	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
-	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
-	Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
-	Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
-	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
-	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
-	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+	if gopts.JSON {
+		err := json.NewEncoder(gopts.stdout).Encode(stats)
+		if err != nil {
+			Warnf("JSON encode failed: %v\n", err)
+		}
+	} else {
+		Printf("\n")
+		Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
+		Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
+		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
+		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
+		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
+		Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
+		Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+	}
 
 	return nil
 }

cmd/restic/cmd_dump.go

@@ -67,41 +67,31 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
-	if tree == nil {
-		return fmt.Errorf("called with a nil tree")
-	}
-	if repo == nil {
-		return fmt.Errorf("called with a nil repository")
-	}
-	l := len(pathComponents)
-	if l == 0 {
-		return fmt.Errorf("empty path components")
-	}
-
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
 		if err := checkStdoutArchive(); err != nil {
 			return err
 		}
-		return writeDump(ctx, repo, tree, "/", os.Stdout)
+		return d.DumpTree(ctx, tree, "/")
 	}
 
 	item := filepath.Join(prefix, pathComponents[0])
+	l := len(pathComponents)
 	for _, node := range tree.Nodes {
 		// If dumping something in the highest level it will just take the
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {
 			switch {
 			case l == 1 && dump.IsFile(node):
-				return dump.GetNodeData(ctx, os.Stdout, repo, node)
+				return d.WriteNode(ctx, node)
 			case l > 1 && dump.IsDir(node):
 				subtree, err := repo.LoadTree(ctx, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
 			case dump.IsDir(node):
 				if err := checkStdoutArchive(); err != nil {
 					return err
@@ -110,7 +100,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return err
 				}
-				return writeDump(ctx, repo, subtree, item, os.Stdout)
+				return d.DumpTree(ctx, subtree, item)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
 			case !dump.IsFile(node):
@@ -128,12 +118,8 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 
-	var wd dump.WriteDump
 	switch opts.Archive {
-	case "tar":
-		wd = dump.WriteTar
-	case "zip":
-		wd = dump.WriteZip
+	case "tar", "zip":
 	default:
 		return fmt.Errorf("unknown archive format %q", opts.Archive)
 	}
@@ -166,7 +152,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -187,7 +173,8 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
+	d := dump.New(opts.Archive, repo, os.Stdout)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
 	if err != nil {
 		Exitf(2, "cannot dump file: %v", err)
 	}

cmd/restic/cmd_find.go

@@ -267,7 +267,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}
@@ -351,7 +351,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
 			return false, walker.ErrSkipNode
 		}

cmd/restic/cmd_forget.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"io"
 
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -15,8 +16,9 @@ var cmdForget = &cobra.Command{
 	Long: `
 The "forget" command removes snapshots according to a policy. Please note that
 this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove this (now unreferenced)
-data after 'forget' was run successfully, see the 'prune' command.
+is a reference to data stored there. In order to remove the unreferenced data
+after "forget" was run successfully, see the "prune" command. Please also read
+the documentation for "forget" to learn about important security considerations.
 
 EXIT STATUS
 ===========
@@ -108,10 +110,16 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(gopts.ctx, repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
+	if gopts.NoLock && !opts.DryRun {
+		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
+	}
+
+	if !opts.DryRun || !gopts.NoLock {
+		lock, err := lockRepoExclusive(gopts.ctx, repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
 	}
 
 	ctx, cancel := context.WithCancel(gopts.ctx)

cmd/restic/cmd_key.go

@@ -131,6 +131,11 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
+	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
 	Verbosef("saved new key as %s\n", id)
 
 	return nil
@@ -161,8 +166,14 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
+	oldID := repo.KeyName()
 
-	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
+	h := restic.Handle{Type: restic.KeyFile, Name: oldID}
 	err = repo.Backend().Remove(gopts.ctx, h)
 	if err != nil {
 		return err
@@ -173,6 +184,19 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	return nil
 }
 
+func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
+	// Verify new key to make sure it really works. A broken key can render the
+	// whole repository inaccessible
+	err := repo.SearchKey(ctx, pw, 0, key.Name())
+	if err != nil {
+		// the key is invalid, try to remove it
+		h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
+		_ = repo.Backend().Remove(ctx, h)
+		return errors.Fatalf("failed to access repository with new key: %v", err)
+	}
+	return nil
+}
+
 func runKey(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
 		return errors.Fatal("wrong number of arguments")

cmd/restic/cmd_list.go

@@ -39,7 +39,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if !opts.NoLock {
+	if !opts.NoLock && args[0] != "locks" {
 		lock, err := lockRepo(opts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {

cmd/restic/cmd_ls.go

@@ -61,9 +61,9 @@ func init() {
 
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
-	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 }
 
@@ -77,31 +77,33 @@ type lsSnapshot struct {
 // Print node in our custom JSON format, followed by a newline.
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	n := &struct {
-		Name       string      `json:"name"`
-		Type       string      `json:"type"`
-		Path       string      `json:"path"`
-		UID        uint32      `json:"uid"`
-		GID        uint32      `json:"gid"`
-		Size       *uint64     `json:"size,omitempty"`
-		Mode       os.FileMode `json:"mode,omitempty"`
-		ModTime    time.Time   `json:"mtime,omitempty"`
-		AccessTime time.Time   `json:"atime,omitempty"`
-		ChangeTime time.Time   `json:"ctime,omitempty"`
-		StructType string      `json:"struct_type"` // "node"
+		Name        string      `json:"name"`
+		Type        string      `json:"type"`
+		Path        string      `json:"path"`
+		UID         uint32      `json:"uid"`
+		GID         uint32      `json:"gid"`
+		Size        *uint64     `json:"size,omitempty"`
+		Mode        os.FileMode `json:"mode,omitempty"`
+		Permissions string      `json:"permissions,omitempty"`
+		ModTime     time.Time   `json:"mtime,omitempty"`
+		AccessTime  time.Time   `json:"atime,omitempty"`
+		ChangeTime  time.Time   `json:"ctime,omitempty"`
+		StructType  string      `json:"struct_type"` // "node"
 
 		size uint64 // Target for Size pointer.
 	}{
-		Name:       node.Name,
-		Type:       node.Type,
-		Path:       path,
-		UID:        node.UID,
-		GID:        node.GID,
-		size:       node.Size,
-		Mode:       node.Mode,
-		ModTime:    node.ModTime,
-		AccessTime: node.AccessTime,
-		ChangeTime: node.ChangeTime,
-		StructType: "node",
+		Name:        node.Name,
+		Type:        node.Type,
+		Path:        path,
+		UID:         node.UID,
+		GID:         node.GID,
+		size:        node.Size,
+		Mode:        node.Mode,
+		Permissions: node.Mode.String(),
+		ModTime:     node.ModTime,
+		AccessTime:  node.AccessTime,
+		ChangeTime:  node.ChangeTime,
+		StructType:  "node",
 	}
 	// Always print size for regular files, even when empty,
 	// but never for other types.
@@ -114,7 +116,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatal("no snapshot ID specified")
+		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
 
 	// extract any specific directories to walk

cmd/restic/cmd_ls_test.go

@@ -18,6 +18,7 @@ func TestLsNodeJSON(t *testing.T) {
 		expect string
 	}{
 		// Mode is omitted when zero.
+		// Permissions, by convention is "-" per mode bit
 		{
 			path: "/bar/baz",
 			Node: restic.Node{
@@ -31,7 +32,7 @@ func TestLsNodeJSON(t *testing.T) {
 				Group: "nobodies",
 				Links: 1,
 			},
-			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 
 		// Even empty files get an explicit size.
@@ -48,7 +49,7 @@ func TestLsNodeJSON(t *testing.T) {
 				Group: "not printed",
 				Links: 0xF00,
 			},
-			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 
 		// Non-regular files do not get a size.
@@ -61,7 +62,7 @@ func TestLsNodeJSON(t *testing.T) {
 				Mode:       os.ModeSymlink | 0777,
 				LinkTarget: "not printed",
 			},
-			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 
 		{
@@ -74,7 +75,7 @@ func TestLsNodeJSON(t *testing.T) {
 				AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
 				ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
 			},
-			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
+			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
 		},
 	} {
 		buf := new(bytes.Buffer)

cmd/restic/cmd_mount.go

@@ -162,7 +162,8 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	root := fuse.NewRoot(repo, cfg)
 
 	Printf("Now serving the repository at %s\n", mountpoint)
-	Printf("When finished, quit with Ctrl-c or umount the mountpoint.\n")
+	Printf("Use another terminal or tool to browse the contents of this folder.\n")
+	Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
 
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, root)

cmd/restic/cmd_prune.go

@@ -66,6 +66,7 @@ func addPruneOptions(c *cobra.Command) {
 }
 
 func verifyPruneOptions(opts *PruneOptions) error {
+	opts.MaxRepackBytes = math.MaxUint64
 	if len(opts.MaxRepackSize) > 0 {
 		size, err := parseSizeStr(opts.MaxRepackSize)
 		if err != nil {
@@ -119,18 +120,6 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 
-func shortenStatus(maxLength int, s string) string {
-	if len(s) <= maxLength {
-		return s
-	}
-
-	if maxLength < 3 {
-		return s[:maxLength]
-	}
-
-	return s[:maxLength-3] + "..."
-}
-
 func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
@@ -430,11 +419,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 
 	for _, p := range repackCandidates {
 		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
-
-		reachedRepackSize := false
-		if opts.MaxRepackBytes > 0 {
-			reachedRepackSize = stats.size.repack+p.unusedSize+p.usedSize > opts.MaxRepackBytes
-		}
+		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
 
 		switch {
 		case reachedRepackSize:
@@ -459,26 +444,26 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		stats.size.repackrm += stats.size.duplicate
 	}
 
-	Verboseff("\nused:        %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
+	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
 	if stats.blobs.duplicate > 0 {
-		Verboseff("duplicates:  %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
+		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
 	}
-	Verboseff("unused:      %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
+	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
 	if stats.size.unref > 0 {
-		Verboseff("unreferenced:                   %s\n", formatBytes(stats.size.unref))
+		Verboseff("unreferenced:                    %s\n", formatBytes(stats.size.unref))
 	}
 	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
 	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
 	unusedSize := stats.size.duplicate + stats.size.unused
-	Verboseff("total:       %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
+	Verboseff("total:        %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
 	Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
 
-	Verbosef("\nto repack:   %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
-	Verbosef("this removes %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
-	Verbosef("to delete:   %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
+	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
+	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
+	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
 	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
-	Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
-	Verbosef("remaining:   %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
+	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
+	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
 	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
 	Verbosef("unused size after prune: %s (%s of remaining size)\n",
 		formatBytes(unusedAfter), formatPercent(unusedAfter, totalSize-totalPruneSize))
@@ -487,11 +472,11 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
 	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
 
-	Verboseff("to keep:   %10d packs\n", stats.packs.keep)
-	Verboseff("to repack: %10d packs\n", len(repackPacks))
-	Verboseff("to delete: %10d packs\n", len(removePacks))
+	Verboseff("to keep:      %10d packs\n", stats.packs.keep)
+	Verboseff("to repack:    %10d packs\n", len(repackPacks))
+	Verboseff("to delete:    %10d packs\n", len(removePacks))
 	if len(removePacksFirst) > 0 {
-		Verboseff("to delete: %10d unreferenced packs\n\n", len(removePacksFirst))
+		Verboseff("to delete:    %10d unreferenced packs\n\n", len(removePacksFirst))
 	}
 
 	if opts.DryRun {

cmd/restic/cmd_rebuild_index.go

@@ -73,7 +73,27 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		}
 	} else {
 		Verbosef("loading indexes...\n")
-		err := repo.LoadIndex(gopts.ctx)
+		mi := repository.NewMasterIndex()
+		err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+			if err != nil {
+				Warnf("removing invalid index %v: %v\n", id, err)
+				obsoleteIndexes = append(obsoleteIndexes, id)
+				return nil
+			}
+
+			mi.Insert(idx)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		err = mi.MergeFinalIndexes()
+		if err != nil {
+			return err
+		}
+
+		err = repo.SetIndex(mi)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_recover.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"time"
 
@@ -11,11 +12,11 @@ import (
 
 var cmdRecover = &cobra.Command{
 	Use:   "recover [flags]",
-	Short: "Recover data from the repository",
+	Short: "Recover data from the repository not referenced by snapshots",
 	Long: `
 The "recover" command builds a new snapshot from all directories it can find in
-the raw data of the repository. It can be used if, for example, a snapshot has
-been removed by accident with "forget".
+the raw data of the repository which are not referenced in an existing snapshot.
+It can be used if, for example, a snapshot has been removed by accident with "forget".
 
 EXIT STATUS
 ===========
@@ -59,24 +60,14 @@ func runRecover(gopts GlobalOptions) error {
 	trees := make(map[restic.ID]bool)
 
 	for blob := range repo.Index().Each(gopts.ctx) {
-		if blob.Blob.Type != restic.TreeBlob {
-			continue
+		if blob.Type == restic.TreeBlob {
+			trees[blob.Blob.ID] = false
 		}
-		trees[blob.Blob.ID] = false
 	}
 
-	cur := 0
-	max := len(trees)
-	Verbosef("load %d trees\n\n", len(trees))
-
+	Verbosef("load %d trees\n", len(trees))
+	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
-		cur++
-		Verbosef("\rtree (%v/%v)", cur, max)
-
-		if !trees[id] {
-			trees[id] = false
-		}
-
 		tree, err := repo.LoadTree(gopts.ctx, id)
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
@@ -84,28 +75,39 @@ func runRecover(gopts GlobalOptions) error {
 		}
 
 		for _, node := range tree.Nodes {
-			if node.Type != "dir" || node.Subtree == nil {
-				continue
+			if node.Type == "dir" && node.Subtree != nil {
+				trees[*node.Subtree] = true
 			}
-
-			subtree := *node.Subtree
-			trees[subtree] = true
 		}
+		bar.Add(1)
 	}
-	Verbosef("\ndone\n")
+	bar.Done()
+
+	Verbosef("load snapshots\n")
+	err = restic.ForAllSnapshots(gopts.ctx, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+		trees[*sn.Tree] = true
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	Verbosef("done\n")
 
 	roots := restic.NewIDSet()
 	for id, seen := range trees {
-		if seen {
-			continue
+		if !seen {
+			Verboseff("found root tree %v\n", id.Str())
+			roots.Insert(id)
 		}
+	}
+	Printf("\nfound %d unreferenced roots\n", len(roots))
 
-		roots.Insert(id)
+	if len(roots) == 0 {
+		Verbosef("no snapshot to write.\n")
+		return nil
 	}
 
-	Verbosef("found %d roots\n", len(roots))
-
-	tree := restic.NewTree()
+	tree := restic.NewTree(len(roots))
 	for id := range roots {
 		var subtreeID = id
 		node := restic.Node{
@@ -117,7 +119,7 @@ func runRecover(gopts GlobalOptions) error {
 			ModTime:    time.Now(),
 			ChangeTime: time.Now(),
 		}
-		err = tree.Insert(&node)
+		err := tree.Insert(&node)
 		if err != nil {
 			return err
 		}
@@ -133,19 +135,23 @@ func runRecover(gopts GlobalOptions) error {
 		return errors.Fatalf("unable to save blobs to the repo: %v", err)
 	}
 
-	sn, err := restic.NewSnapshot([]string{"/recover"}, []string{}, hostname, time.Now())
+	return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
+
+}
+
+func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
+	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
-	sn.Tree = &treeID
+	sn.Tree = tree
 
-	id, err := repo.SaveJSONUnpacked(gopts.ctx, restic.SnapshotFile, sn)
+	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
 	Printf("saved new snapshot %v\n", id.Str())
-
 	return nil
 }

cmd/restic/cmd_restore.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"strings"
+	"time"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
@@ -117,7 +118,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -202,6 +203,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	if opts.Verify {
 		Verbosef("verifying files in %s\n", opts.Target)
 		var count int
+		t0 := time.Now()
 		count, err = res.VerifyFiles(ctx, opts.Target)
 		if err != nil {
 			return err
@@ -209,7 +211,8 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		if totalErrors > 0 {
 			return errors.Fatalf("There were %d errors\n", totalErrors)
 		}
-		Verbosef("finished verifying %d files in %s\n", count, opts.Target)
+		Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target,
+			time.Since(t0).Round(time.Millisecond))
 	}
 
 	return nil

cmd/restic/cmd_self_update.go

@@ -71,7 +71,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
 		}
 	}
 
-	Printf("writing restic to %v\n", opts.Output)
+	Verbosef("writing restic to %v\n", opts.Output)
 
 	v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef)
 	if err != nil {

cmd/restic/find.go

@@ -23,7 +23,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			for _, s := range snapshotIDs {
 				if s == "latest" {
 					usedFilter = true
-					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts)
+					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts, nil)
 					if err != nil {
 						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
 						continue
@@ -31,7 +31,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 				} else {
 					id, err = restic.FindSnapshot(ctx, repo, s)
 					if err != nil {
-						Warnf("Ignoring %q, it is not a snapshot id\n", s)
+						Warnf("Ignoring %q: %v\n", s, err)
 						continue
 					}
 				}

cmd/restic/global.go

@@ -40,7 +40,7 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-var version = "0.12.1"
+var version = "0.13.1"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -61,6 +61,7 @@ type GlobalOptions struct {
 	CacheDir        string
 	NoCache         bool
 	CACerts         []string
+	InsecureTLS     bool
 	TLSClientCert   string
 	CleanupCache    bool
 
@@ -97,6 +98,8 @@ func init() {
 	var cancel context.CancelFunc
 	globalOptions.ctx, cancel = context.WithCancel(context.Background())
 	AddCleanupHandler(func() error {
+		// Must be called before the unlock cleanup handler to ensure that the latter is
+		// not blocked due to limited number of backend connections, see #1434
 		cancel()
 		return nil
 	})
@@ -115,6 +118,7 @@ func init() {
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
 	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "`file` to load root certificates from (default: use system certificates)")
 	f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key")
+	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repo (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
@@ -197,16 +201,21 @@ func restoreTerminal() {
 }
 
 // ClearLine creates a platform dependent string to clear the current
-// line, so it can be overwritten. ANSI sequences are not supported on
-// current windows cmd shell.
-func ClearLine() string {
-	if runtime.GOOS == "windows" {
-		if w := stdoutTerminalWidth(); w > 0 {
-			return strings.Repeat(" ", w-1) + "\r"
-		}
-		return ""
+// line, so it can be overwritten.
+//
+// w should be the terminal width, or 0 to let clearLine figure it out.
+func clearLine(w int) string {
+	if runtime.GOOS != "windows" {
+		return "\x1b[2K"
 	}
-	return "\x1b[2K"
+
+	// ANSI sequences are not supported on Windows cmd shell.
+	if w <= 0 {
+		if w = stdoutTerminalWidth(); w <= 0 {
+			return ""
+		}
+	}
+	return strings.Repeat(" ", w-1) + "\r"
 }
 
 // Printf writes the message to the configured stdout stream.
@@ -247,31 +256,6 @@ func Verboseff(format string, args ...interface{}) {
 	}
 }
 
-// PrintProgress wraps fmt.Printf to handle the difference in writing progress
-// information to terminals and non-terminal stdout
-func PrintProgress(format string, args ...interface{}) {
-	var (
-		message         string
-		carriageControl string
-	)
-	message = fmt.Sprintf(format, args...)
-
-	if !(strings.HasSuffix(message, "\r") || strings.HasSuffix(message, "\n")) {
-		if stdoutCanUpdateStatus() {
-			carriageControl = "\r"
-		} else {
-			carriageControl = "\n"
-		}
-		message = fmt.Sprintf("%s%s", message, carriageControl)
-	}
-
-	if stdoutCanUpdateStatus() {
-		message = fmt.Sprintf("%s%s", ClearLine(), message)
-	}
-
-	fmt.Print(message)
-}
-
 // Warnf writes the message to the configured stderr stream.
 func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
@@ -520,8 +504,9 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 
 	// cleanup old cache dirs if instructed to do so
 	if opts.CleanupCache {
-		Printf("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
-
+		if stdoutIsTerminal() && !opts.JSON {
+			Verbosef("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
+		}
 		for _, item := range oldCacheDirs {
 			dir := filepath.Join(c.Base, item.Name())
 			err = fs.RemoveAll(dir)
@@ -572,6 +557,12 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
 		}
 
+		if cfg.KeyID == "" && cfg.Secret != "" {
+			return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
+		} else if cfg.KeyID != "" && cfg.Secret == "" {
+			return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
+		}
+
 		if cfg.Region == "" {
 			cfg.Region = os.Getenv("AWS_DEFAULT_REGION")
 		}
@@ -691,6 +682,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
+		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -713,7 +705,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	case "azure":
 		be, err = azure.Open(cfg.(azure.Config), rt)
 	case "swift":
-		be, err = swift.Open(cfg.(swift.Config), rt)
+		be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
 	case "b2":
 		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
@@ -771,6 +763,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
+		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -789,7 +782,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "azure":
 		return azure.Create(cfg.(azure.Config), rt)
 	case "swift":
-		return swift.Open(cfg.(swift.Config), rt)
+		return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
 	case "b2":
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":

cmd/restic/integration_helpers_test.go

@@ -67,7 +67,7 @@ func isSymlink(fi os.FileInfo) bool {
 
 func sameModTime(fi1, fi2 os.FileInfo) bool {
 	switch runtime.GOOS {
-	case "darwin", "freebsd", "openbsd", "netbsd":
+	case "darwin", "freebsd", "openbsd", "netbsd", "solaris":
 		if isSymlink(fi1) && isSymlink(fi2) {
 			return true
 		}

cmd/restic/integration_test.go

@@ -15,6 +15,7 @@ import (
 	"regexp"
 	"runtime"
 	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -158,8 +159,11 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
 	buf := bytes.NewBuffer(nil)
 
 	globalOptions.stdout = buf
+	oldStdout := gopts.stdout
+	gopts.stdout = buf
 	defer func() {
 		globalOptions.stdout = os.Stdout
+		gopts.stdout = oldStdout
 	}()
 
 	opts := DiffOptions{
@@ -345,6 +349,57 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 }
 
+func TestDryRunBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	dryOpts := BackupOptions{DryRun: true}
+
+	// dry run before first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(snapshotIDs) == 0,
+		"expected no snapshot, got %v", snapshotIDs)
+	packIDs := testRunList(t, "packs", env.gopts)
+	rtest.Assert(t, len(packIDs) == 0,
+		"expected no data, got %v", snapshotIDs)
+	indexIDs := testRunList(t, "index", env.gopts)
+	rtest.Assert(t, len(indexIDs) == 0,
+		"expected no index, got %v", snapshotIDs)
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// dry run between backups
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter := testRunList(t, "snapshots", env.gopts)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter := testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter := testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// another dry run
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter = testRunList(t, "snapshots", env.gopts)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter = testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter = testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+}
+
 func TestBackupNonExistingFile(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -374,10 +429,11 @@ func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, rem
 
 	// Get all tree packs
 	rtest.OK(t, r.LoadIndex(gopts.ctx))
+
 	treePacks := restic.NewIDSet()
-	for _, idx := range r.Index().(*repository.MasterIndex).All() {
-		for _, id := range idx.TreePacks() {
-			treePacks.Insert(id)
+	for pb := range r.Index().Each(context.TODO()) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
 		}
 	}
 
@@ -436,11 +492,10 @@ func TestBackupTreeLoadError(t *testing.T) {
 	r, err := OpenRepository(env.gopts)
 	rtest.OK(t, err)
 	rtest.OK(t, r.LoadIndex(env.gopts.ctx))
-	// collect tree packs of subdirectory
-	subTreePacks := restic.NewIDSet()
-	for _, idx := range r.Index().(*repository.MasterIndex).All() {
-		for _, id := range idx.TreePacks() {
-			subTreePacks.Insert(id)
+	treePacks := restic.NewIDSet()
+	for pb := range r.Index().Each(context.TODO()) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
 		}
 	}
 
@@ -448,7 +503,7 @@ func TestBackupTreeLoadError(t *testing.T) {
 	testRunCheck(t, env.gopts)
 
 	// delete the subdirectory pack first
-	for id := range subTreePacks {
+	for id := range treePacks {
 		rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}))
 	}
 	testRunRebuildIndex(t, env.gopts)
@@ -1033,6 +1088,41 @@ func TestKeyAddRemove(t *testing.T) {
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
 }
 
+type emptySaveBackend struct {
+	restic.Backend
+}
+
+func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+	return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
+}
+
+func TestKeyProblems(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+		return &emptySaveBackend{r}, nil
+	}
+
+	testKeyNewPassword = "geheim2"
+	defer func() {
+		testKeyNewPassword = ""
+	}()
+
+	err := runKey(env.gopts, []string{"passwd"})
+	t.Log(err)
+	rtest.Assert(t, err != nil, "expected passwd change to fail")
+
+	err = runKey(env.gopts, []string{"add"})
+	t.Log(err)
+	rtest.Assert(t, err != nil, "expected key adding to fail")
+
+	t.Logf("testing access with initial password %q\n", env.gopts.password)
+	rtest.OK(t, runKey(env.gopts, []string{"list"}))
+	testRunCheck(t, env.gopts)
+}
+
 func testFileSize(filename string, size int64) error {
 	fi, err := os.Stat(filename)
 	if err != nil {
@@ -1330,7 +1420,7 @@ func TestFindJSON(t *testing.T) {
 	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
 }
 
-func TestRebuildIndex(t *testing.T) {
+func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 
@@ -1350,8 +1440,10 @@ func TestRebuildIndex(t *testing.T) {
 		t.Fatalf("did not find hint for rebuild-index command")
 	}
 
+	env.gopts.backendTestHook = backendTestHook
 	testRunRebuildIndex(t, env.gopts)
 
+	env.gopts.backendTestHook = nil
 	out, err = testRunCheckOutput(env.gopts)
 	if len(out) != 0 {
 		t.Fatalf("expected no output from the checker, got: %v", out)
@@ -1362,9 +1454,57 @@ func TestRebuildIndex(t *testing.T) {
 	}
 }
 
+func TestRebuildIndex(t *testing.T) {
+	testRebuildIndex(t, nil)
+}
+
 func TestRebuildIndexAlwaysFull(t *testing.T) {
+	indexFull := repository.IndexFull
+	defer func() {
+		repository.IndexFull = indexFull
+	}()
 	repository.IndexFull = func(*repository.Index) bool { return true }
-	TestRebuildIndex(t)
+	testRebuildIndex(t, nil)
+}
+
+// indexErrorBackend modifies the first index after reading.
+type indexErrorBackend struct {
+	restic.Backend
+	lock     sync.Mutex
+	hasErred bool
+}
+
+func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+	return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
+		// protect hasErred
+		b.lock.Lock()
+		defer b.lock.Unlock()
+		if !b.hasErred && h.Type == restic.IndexFile {
+			b.hasErred = true
+			return consumer(errorReadCloser{rd})
+		}
+		return consumer(rd)
+	})
+}
+
+type errorReadCloser struct {
+	io.Reader
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.Reader.Read(p)
+	if n > 0 {
+		p[0] ^= 1
+	}
+	return n, err
+}
+
+func TestRebuildIndexDamage(t *testing.T) {
+	testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
+		return &indexErrorBackend{
+			Backend: r,
+		}, nil
+	})
 }
 
 type appendOnlyBackend struct {
@@ -1607,7 +1747,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 
 // a listOnceBackend only allows listing once per filetype
 // listing filetypes more than once may cause problems with eventually consistent
-// backends (like e.g. AWS S3) as the second listing may be inconsistent to what
+// backends (like e.g. Amazon S3) as the second listing may be inconsistent to what
 // is expected by the first listing + some operations.
 type listOnceBackend struct {
 	restic.Backend
@@ -1835,10 +1975,8 @@ var diffOutputRegexPatterns = []string{
 	"Removed: +2[0-9]{2}\\.[0-9]{3} KiB",
 }
 
-func TestDiff(t *testing.T) {
+func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
 	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
 	testRunInit(t, env.gopts)
 
 	datadir := filepath.Join(env.base, "testdata")
@@ -1874,19 +2012,82 @@ func TestDiff(t *testing.T) {
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	_, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 
+	return env, cleanup, firstSnapshotID, secondSnapshotID
+}
+
+func TestDiff(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
 	_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
 	rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
 
 	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	if err != nil {
-		t.Fatalf("expected no error from diff for test repository, got %v", err)
-	}
+	rtest.OK(t, err)
 
 	for _, pattern := range diffOutputRegexPatterns {
 		r, err := regexp.Compile(pattern)
 		rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern)
 		rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out)
 	}
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
+}
+
+type typeSniffer struct {
+	MessageType string `json:"message_type"`
+}
+
+func TestDiffJSON(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
+	env.gopts.JSON = true
+	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	var stat DiffStatsContainer
+	var changes int
+
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := scanner.Text()
+		var sniffer typeSniffer
+		rtest.OK(t, json.Unmarshal([]byte(line), &sniffer))
+		switch sniffer.MessageType {
+		case "change":
+			changes++
+		case "statistics":
+			rtest.OK(t, json.Unmarshal([]byte(line), &stat))
+		default:
+			t.Fatalf("unexpected message type %v", sniffer.MessageType)
+		}
+	}
+	rtest.Equals(t, 9, changes)
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	stat = DiffStatsContainer{}
+	rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat))
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+	rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids")
 }
 
 type writeToOnly struct {

cmd/restic/lock.go

@@ -16,6 +16,7 @@ var globalLocks struct {
 	cancelRefresh chan struct{}
 	refreshWG     sync.WaitGroup
 	sync.Mutex
+	sync.Once
 }
 
 func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
@@ -27,6 +28,12 @@ func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*resti
 }
 
 func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
+	// make sure that a repository is unlocked properly and after cancel() was
+	// called by the cleanup handler in global.go
+	globalLocks.Do(func() {
+		AddCleanupHandler(unlockAll)
+	})
+
 	lockFn := restic.NewLock
 	if exclusive {
 		lockFn = restic.NewExclusiveLock
@@ -128,7 +135,3 @@ func unlockAll() error {
 
 	return nil
 }
-
-func init() {
-	AddCleanupHandler(unlockAll)
-}

cmd/restic/progress.go

@@ -4,15 +4,17 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/restic/restic/internal/ui/progress"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 // calculateProgressInterval returns the interval configured via RESTIC_PROGRESS_FPS
 // or if unset returns an interval for 60fps on interactive terminals and 0 (=disabled)
 // for non-interactive terminals or when run using the --quiet flag
-func calculateProgressInterval(show bool) time.Duration {
+func calculateProgressInterval(show bool, json bool) time.Duration {
 	interval := time.Second / 60
 	fps, err := strconv.ParseFloat(os.Getenv("RESTIC_PROGRESS_FPS"), 64)
 	if err == nil && fps > 0 {
@@ -20,7 +22,7 @@ func calculateProgressInterval(show bool) time.Duration {
 			fps = 60
 		}
 		interval = time.Duration(float64(time.Second) / fps)
-	} else if !stdoutCanUpdateStatus() || !show {
+	} else if !json && !stdoutCanUpdateStatus() || !show {
 		interval = 0
 	}
 	return interval
@@ -31,7 +33,8 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 	if !show {
 		return nil
 	}
-	interval := calculateProgressInterval(show)
+	interval := calculateProgressInterval(show, false)
+	canUpdateStatus := stdoutCanUpdateStatus()
 
 	return progress.New(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
 		var status string
@@ -42,13 +45,36 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 				formatDuration(d), formatPercent(v, max), v, max, description)
 		}
 
-		if w := stdoutTerminalWidth(); w > 0 {
-			status = shortenStatus(w, status)
-		}
-
-		PrintProgress("%s", status)
+		printProgress(status, canUpdateStatus)
 		if final {
 			fmt.Print("\n")
 		}
 	})
 }
+
+func printProgress(status string, canUpdateStatus bool) {
+	w := stdoutTerminalWidth()
+	if w > 0 {
+		if w < 3 {
+			status = termstatus.Truncate(status, w)
+		} else {
+			status = termstatus.Truncate(status, w-3) + "..."
+		}
+	}
+
+	var carriageControl, clear string
+
+	if canUpdateStatus {
+		clear = clearLine(w)
+	}
+
+	if !(strings.HasSuffix(status, "\r") || strings.HasSuffix(status, "\n")) {
+		if canUpdateStatus {
+			carriageControl = "\r"
+		} else {
+			carriageControl = "\n"
+		}
+	}
+
+	_, _ = os.Stdout.Write([]byte(clear + status + carriageControl))
+}

doc/020_installation.rst

@@ -131,7 +131,14 @@ On openSUSE (leap 15.0 and greater, and tumbleweed), you can install restic usin
 RHEL & CentOS
 =============
 
-restic can be installed via copr repository, for RHEL7/CentOS you can try the following:
+For RHEL / CentOS Stream 8 & 9 restic can be installed from the EPEL repository:
+
+.. code-block:: console
+
+    $ dnf install epel-release
+    $ dnf install restic
+
+For RHEL7/CentOS there is a copr repository available, you can try the following:
 
 .. code-block:: console
 
@@ -188,8 +195,20 @@ are considered stable and releases are made regularly in a controlled manner.
 There's both pre-compiled binaries for different platforms as well as the source
 code available for download. Just download and run the one matching your system.
 
-The official binaries can be updated in place using the ``restic self-update``
-command (needs restic 0.9.3 or later):
+On your first installation, if you desire, you can verify the integrity of your
+downloads by testing the SHA-256 checksums listed in ``SHA256SUMS`` and verifying
+the integrity of the file ``SHA256SUMS`` with the PGP signature in ``SHA256SUMS.asc``. 
+The PGP signature was created using the key (`0x91A6868BD3F7A907 <https://restic.net/gpg-key-alex.asc>`__):
+
+::
+
+    pub   4096R/91A6868BD3F7A907 2014-11-01
+          Key fingerprint = CF8F 18F2 8445 7597 3F79  D4E1 91A6 868B D3F7 A907
+          uid                          Alexander Neumann <alexander@bumpern.de>
+          sub   4096R/D5FC2ACF4043FDF1 2014-11-01
+
+Once downloaded, the official binaries can be updated in place using the 
+``restic self-update`` command (needs restic 0.9.3 or later):
 
 .. code-block:: console
 
@@ -255,7 +274,7 @@ From Source
 ***********
 
 restic is written in the Go programming language and you need at least
-Go version 1.13. Building restic may also work with older versions of Go,
+Go version 1.14. Building restic may also work with older versions of Go,
 but that's not supported. See the `Getting
 started <https://golang.org/doc/install>`__ guide of the Go project for
 instructions how to install Go.
@@ -283,6 +302,8 @@ supply the target OS and platform via the command-line options like this
 
     $ go run build.go --goos linux --goarch arm --goarm 6
 
+    $ go run build.go --goos solaris --goarch amd64
+
 The resulting binary is statically linked and does not require any
 libraries.
 

doc/030_preparing_a_new_repo.rst

@@ -257,7 +257,7 @@ Minio Server
 ************
 
 `Minio <https://www.minio.io>`__ is an Open Source Object Storage,
-written in Go and compatible with AWS S3 API.
+written in Go and compatible with Amazon S3 API.
 
 -  Download and Install `Minio
    Server <https://minio.io/downloads/#minio-server>`__.
@@ -287,7 +287,7 @@ this command.
 Wasabi
 ************
 
-`Wasabi <https://wasabi.com>`__ is a low cost AWS S3 conformant object storage provider.
+`Wasabi <https://wasabi.com>`__ is a low cost Amazon S3 conformant object storage provider.
 Due to it's S3 conformance, Wasabi can be used as a storage provider for a restic repository.
 
 -  Create a Wasabi bucket using the `Wasabi Console <https://console.wasabisys.com>`__.
@@ -442,6 +442,14 @@ dashboard on the "Buckets" page when signed into your B2 account:
     $ export B2_ACCOUNT_ID=<MY_APPLICATION_KEY_ID>
     $ export B2_ACCOUNT_KEY=<MY_APPLICATION_KEY>
 
+To get application keys, a user can go to the App Keys section of the Backblaze
+account portal.  You must create a master application key first.  From there, you
+can generate a standard Application Key.  Please note that the Application Key
+should be treated like a password and will only appear once.  If an Application
+Key is forgotten, you must generate a new one.
+
+For more information on application keys, refer to the Backblaze `documentation <https://www.backblaze.com/b2/docs/application_keys.html>`__.
+
 .. note:: As of version 0.9.2, restic supports both master and non-master `application keys <https://www.backblaze.com/b2/docs/application_keys.html>`__. If using a non-master application key, ensure that it is created with at least **read and write** access to the B2 bucket. On earlier versions of restic, a master application key is required.
 
 You can then initialize a repository stored at Backblaze B2. If the
@@ -607,10 +615,11 @@ configuring a bandwidth limit for rclone can be achieved by setting the
 
 For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``.
 
-The rclone backend has two additional options:
+The rclone backend has three additional options:
 
  * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone``
  * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete``
+ * ``-o rclone.timeout`` specifies timeout for waiting on repository opening, the default value is ``1m``
 
 The reason for the ``--b2-hard-delete`` parameters can be found in the corresponding GitHub `issue #1657`_.
 
@@ -648,7 +657,8 @@ credentials) is encrypted/decrypted locally, then sent/received via
 A more advanced version of this setup forbids specific hosts from removing
 files in a repository. See the `blog post by Simon Ruderich
 <https://ruderich.org/simon/notes/append-only-backups-with-restic-and-rclone>`_
-for details.
+for details and the documentation for the ``forget`` command to learn about
+important security considerations.
 
 The rclone command may also be hard-coded in the SSH configuration or the
 user's public key, in this case it may be sufficient to just start the SSH
@@ -674,7 +684,7 @@ interaction. If you use emulation environments like
 ``Mintty`` or ``rxvt``, you may get a password error.
 
 You can workaround this by using a special tool called ``winpty`` (look
-`here <https://github.com/msys2/msys2/wiki/Porting>`__ and
+`here <https://www.msys2.org/wiki/Porting/>`__ and
 `here <https://github.com/rprichard/winpty>`__ for detail information).
 On MSYS2, you can install ``winpty`` as follows:
 

doc/040_backup.rst

@@ -187,6 +187,23 @@ On **Windows**, a file is considered unchanged when its path, size
 and modification time match, and only ``--force`` has any effect.
 The other options are recognized but ignored.
 
+Dry Runs
+********
+
+You can perform a backup in dry run mode to see what would happen without
+modifying the repo.
+
+-  ``--dry-run``/``-n`` Report what would be done, without writing to the repository
+
+Combined with ``--verbose``, you can see a list of changes:
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo backup ~/work --dry-run -vv | grep "added"
+    modified  /plan.txt, saved in 0.000s (9.110 KiB added)
+    modified  /archive.tar.gz, saved in 0.140s (25.542 MiB added)
+    Would be added to the repo: 25.551 MiB
+
 Excluding Files
 ***************
 
@@ -272,6 +289,28 @@ On most Unixy shells, you can either quote or use backslashes. For example:
  * ``--exclude="foo bar star/foo.txt"``
  * ``--exclude=foo\ bar\ star/foo.txt``
 
+If a pattern starts with exclamation mark and matches a file that
+was previously matched by a regular pattern, the match is cancelled.
+It works similarly to ``gitignore``, with the same limitation: once a
+directory is excluded, it is not possible to include files inside the
+directory. Here is a complete example to backup a selection of
+directories inside the home directory. It works by excluding any
+directory, then selectively add back some of them.
+
+::
+
+    $HOME/**/*
+    !$HOME/Documents
+    !$HOME/code
+    !$HOME/.emacs.d
+    !$HOME/games
+    # [...]
+    node_modules
+    *~
+    *.o
+    *.lo
+    *.pyc
+
 By specifying the option ``--one-file-system`` you can instruct restic
 to only backup files from the file systems the initially specified files
 or directories reside on. In other words, it will prevent restic from crossing
@@ -420,6 +459,14 @@ written, and the next backup needs to write new metadata again. If you really
 want to save the access time for files and directories, you can pass the
 ``--with-atime`` option to the ``backup`` command.
 
+Note that ``restic`` does not back up some metadata associated with files. Of
+particular note are::
+
+  - file creation date on Unix platforms
+  - inode flags on Unix platforms
+  - file ownership and ACLs on Windows
+  - the "hidden" flag on Windows
+
 Reading data from stdin
 ***********************
 
@@ -466,6 +513,16 @@ The tags can later be used to keep (or forget) snapshots with the ``forget``
 command. The command ``tag`` can be used to modify tags on an existing
 snapshot.
 
+Scheduling backups
+******************
+
+Restic does not have a built-in way of scheduling backups, as it's a tool
+that runs when executed rather than a daemon. There are plenty of different
+ways to schedule backup runs on various different platforms, e.g. systemd
+and cron on Linux/BSD and Task Scheduler in Windows, depending on one's
+needs and requirements. When scheduling restic to run recurringly, please
+make sure to detect already running instances before starting the backup.
+
 Space requirements
 ******************
 
@@ -502,6 +559,8 @@ environment variables. The following lists these environment variables:
     AWS_ACCESS_KEY_ID                   Amazon S3 access key ID
     AWS_SECRET_ACCESS_KEY               Amazon S3 secret access key
     AWS_DEFAULT_REGION                  Amazon S3 default region
+    AWS_PROFILE                         Amazon credentials profile (alternative to specifying key and region)
+    AWS_SHARED_CREDENTIALS_FILE         Location of the AWS CLI shared credentials file (default: ~/.aws/credentials)
 
     ST_AUTH                             Auth URL for keystone v1 authentication
     ST_USER                             Username for keystone v1 authentication

doc/045_working_with_repos.rst

@@ -248,12 +248,12 @@ integrity of the pack files in the repository, use the ``--read-data`` flag:
     repository, beware that it might incur higher bandwidth costs than usual
     and also that it takes more time than the default ``check``.
 
-Alternatively, use the ``--read-data-subset`` parameter to check only a
-subset of the repository pack files at a time. It supports two ways to select a
-subset. One selects a specific range of pack files, the other selects a random
-percentage of pack files.
+Alternatively, use the ``--read-data-subset`` parameter to check only a subset
+of the repository pack files at a time. It supports three ways to select a
+subset. One selects a specific part of pack files, the second and third
+selects a random subset of the pack files by the given percentage or size.
 
-Use ``--read-data-subset=n/t`` to check only a subset of the repository pack
+Use ``--read-data-subset=n/t`` to check a specific part of the repository pack
 files at a time. The parameter takes two values, ``n`` and ``t``. When the check
 command runs, all pack files in the repository are logically divided in ``t``
 (roughly equal) groups, and only files that belong to group number ``n`` are
@@ -268,20 +268,33 @@ over 5 separate invocations:
     $ restic -r /srv/restic-repo check --read-data-subset=4/5
     $ restic -r /srv/restic-repo check --read-data-subset=5/5
 
-Use ``--read-data-subset=n%`` to check a randomly choosen subset of the
-repository pack files. It takes one parameter, ``n``, the percentage of pack
-files to check as an integer or floating point number. This will not guarantee
-to cover all available pack files after sufficient runs, but it is easy to
-automate checking a small subset of data after each backup. For a floating point
-value the following command may be used:
+Use ``--read-data-subset=x%`` to check a randomly choosen subset of the
+repository pack files. It takes one parameter, ``x``, the percentage of
+pack files to check as an integer or floating point number. This will not
+guarantee to cover all available pack files after sufficient runs, but it is
+easy to automate checking a small subset of data after each backup. For a
+floating point value the following command may be used:
 
 .. code-block:: console
 
     $ restic -r /srv/restic-repo check --read-data-subset=2.5%
 
-When checking bigger subsets you most likely specify the percentage as an
-integer:
+When checking bigger subsets you most likely want to specify the percentage
+as an integer:
 
 .. code-block:: console
 
     $ restic -r /srv/restic-repo check --read-data-subset=10%
+
+Use ``--read-data-subset=nS`` to check a randomly chosen subset of the
+repository pack files. It takes one parameter, ``nS``, where 'n' is a whole
+number representing file size and 'S' is the unit of file size (K/M/G/T) of
+pack files to check. Behind the scenes, the specified size will be converted
+to percentage of the total repository size. The behaviour of the check command
+following this conversion will be the same as the percentage option above. For
+a file size value the following command may be used:
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo check --read-data-subset=50M
+    $ restic -r /srv/restic-repo check --read-data-subset=10G

doc/050_restore.rst

@@ -69,7 +69,8 @@ command to serve the repository with FUSE:
     $ restic -r /srv/restic-repo mount /mnt/restic
     enter password for repository:
     Now serving /srv/restic-repo at /mnt/restic
-    When finished, quit with Ctrl-c or umount the mountpoint.
+    Use another terminal or tool to browse the contents of this folder.
+    When finished, quit with Ctrl-c here or umount the mountpoint.
 
 Mounting repositories via FUSE is only possible on Linux, macOS and FreeBSD.
 On Linux, the ``fuse`` kernel module needs to be loaded and the ``fusermount``

doc/060_forget.rst

@@ -14,17 +14,16 @@
 Removing backup snapshots
 #########################
 
-All backup space is finite, so restic allows removing old snapshots.
-This can be done either manually (by specifying a snapshot ID to remove)
-or by using a policy that describes which snapshots to forget. For all
-remove operations, two commands need to be called in sequence:
-``forget`` to remove a snapshot and ``prune`` to actually remove the
-data that was referenced by the snapshot from the repository. This can
-be automated with the ``--prune`` option of the ``forget`` command,
-which runs ``prune`` automatically if snapshots have been removed.
+All backup space is finite, so restic allows removing old snapshots. This can
+be done either manually (by specifying a snapshot ID to remove) or by using a
+policy that describes which snapshots to forget. For all remove operations, two
+commands need to be called in sequence: ``forget`` to remove snapshots, and
+``prune`` to remove the remaining data that was referenced only by the removed
+snapshots. The latter can be automated with the ``--prune`` option of ``forget``,
+which runs ``prune`` automatically if any snapshots were actually removed.
 
 Pruning snapshots can be a time-consuming process, depending on the
-amount of snapshots and data to process. During a prune operation, the
+number of snapshots and data to process. During a prune operation, the
 repository is locked and backups cannot be completed. Please plan your
 pruning so that there's time to complete it and it doesn't interfere with
 regular backup runs.
@@ -90,11 +89,11 @@ command must be run:
     collecting packs for deletion and repacking
     [0:00] 100.00%  5 / 5 packs processed
     
-    to repack:           69 blobs / 1.078 MiB
-    this removes         67 blobs / 1.047 MiB
-    to delete:            7 blobs / 25.726 KiB
-    total prune:         74 blobs / 1.072 MiB
-    remaining:           16 blobs / 38.003 KiB
+    to repack:            69 blobs / 1.078 MiB
+    this removes:         67 blobs / 1.047 MiB
+    to delete:             7 blobs / 25.726 KiB
+    total prune:          74 blobs / 1.072 MiB
+    remaining:            16 blobs / 38.003 KiB
     unused size after prune: 0 B (0.00% of remaining size)
     
     repacking packs
@@ -156,67 +155,75 @@ to ``forget``:
 Removing snapshots according to a policy
 ****************************************
 
-Removing snapshots manually is tedious and error-prone, therefore restic
-allows specifying which snapshots should be removed automatically
-according to a policy. You can specify how many hourly, daily, weekly,
-monthly and yearly snapshots to keep, any other snapshots are removed.
-The most important command-line parameter here is ``--dry-run`` which
-instructs restic to not remove anything but print which snapshots would
-be removed.
+Removing snapshots manually is tedious and error-prone, therefore restic allows
+specifying a policy (one or more ``--keep-*`` options) for which snapshots to
+keep. You can for example define how many hourly, daily, weekly, monthly and
+yearly snapshots to keep, and any other snapshots will be removed.
 
-When ``forget`` is run with a policy, restic loads the list of all
-snapshots, then groups these by host name and list of directories. The grouping
-options can be set with ``--group-by``, to only group snapshots by paths and
-tags use ``--group-by paths,tags``. The policy is then applied to each group of
-snapshots separately. This is a safety feature.
+.. warning:: If you use an append-only repository with policy-based snapshot
+    removal, some security considerations are important. Please refer to the
+    section below for more information.
 
-The ``forget`` command accepts the following parameters:
+.. note:: You can always use the ``--dry-run`` option of the ``forget`` command,
+    which instructs restic to not remove anything but instead just print what
+    actions would be performed.
 
--  ``--keep-last n`` never delete the ``n`` last (most recent) snapshots
--  ``--keep-hourly n`` for the last ``n`` hours in which a snapshot was
-   made, keep only the last snapshot for each hour.
+The ``forget`` command accepts the following policy options:
+
+-  ``--keep-last n`` keep the ``n`` last (most recent) snapshots.
+-  ``--keep-hourly n`` for the last ``n`` hours which have one or more
+   snapshots, keep only the most recent one for each hour.
 -  ``--keep-daily n`` for the last ``n`` days which have one or more
-   snapshots, only keep the last one for that day.
+   snapshots, keep only the most recent one for each day.
 -  ``--keep-weekly n`` for the last ``n`` weeks which have one or more
-   snapshots, only keep the last one for that week.
+   snapshots, keep only the most recent one for each week.
 -  ``--keep-monthly n`` for the last ``n`` months which have one or more
-   snapshots, only keep the last one for that month.
+   snapshots, keep only the most recent one for each month.
 -  ``--keep-yearly n`` for the last ``n`` years which have one or more
-   snapshots, only keep the last one for that year.
+   snapshots, keep only the most recent one for each year.
 -  ``--keep-tag`` keep all snapshots which have all tags specified by
    this option (can be specified multiple times).
--  ``--keep-within duration`` keep all snapshots which have been made within
-   the duration of the latest snapshot. ``duration`` needs to be a number of
-   years, months, days, and hours, e.g. ``2y5m7d3h`` will keep all snapshots
-   made in the two years, five months, seven days, and three hours before the
-   latest snapshot.
--  ``--keep-within-hourly duration`` keep all hourly snapshots made within
-   specified duration of the latest snapshot. The duration is specified in 
-   the same way as for ``--keep-within`` and the method for determining
-   hourly snapshots is the same as for ``--keep-hourly``.
--  ``--keep-within-daily duration`` keep all daily snapshots made within
+-  ``--keep-within duration`` keep all snapshots having a timestamp within
+   the specified duration of the latest snapshot, where ``duration`` is a
+   number of years, months, days, and hours. E.g. ``2y5m7d3h`` will keep all
+   snapshots made in the two years, five months, seven days and three hours
+   before the latest (most recent) snapshot.
+-  ``--keep-within-hourly duration`` keep all hourly snapshots made within the
+   specified duration of the latest snapshot. The ``duration`` is specified in
+   the same way as for ``--keep-within`` and the method for determining hourly
+   snapshots is the same as for ``--keep-hourly``.
+-  ``--keep-within-daily duration`` keep all daily snapshots made within the
    specified duration of the latest snapshot.
--  ``--keep-within-weekly duration`` keep all weekly snapshots made within
+-  ``--keep-within-weekly duration`` keep all weekly snapshots made within the
    specified duration of the latest snapshot.
--  ``--keep-within-monthly duration`` keep all monthly snapshots made within
+-  ``--keep-within-monthly duration`` keep all monthly snapshots made within the
    specified duration of the latest snapshot.
--  ``--keep-within-yearly duration`` keep all yearly snapshots made within
+-  ``--keep-within-yearly duration`` keep all yearly snapshots made within the
    specified duration of the latest snapshot.
 
-.. note:: All calendar related ``--keep-*`` options work on the natural time
-    boundaries and not relative to when you run the ``forget`` command. Weeks
-    are Monday 00:00 -> Sunday 23:59, days 00:00 to 23:59, hours :00 to :59, etc.
+.. note:: All calendar related options (``--keep-{hourly,daily,...}``) work on
+    natural time boundaries and *not* relative to when you run ``forget``. Weeks
+    are Monday 00:00 to Sunday 23:59, days 00:00 to 23:59, hours :00 to :59, etc.
+    They also only count hours/days/weeks/etc which have one or more snapshots.
+
+.. note:: All duration related options (``--keep-{within,-*}``) ignore snapshots
+    with a timestamp in the future (relative to when the ``forget`` command is
+    run) and these snapshots will hence not be removed.
 
 .. note:: Specifying ``--keep-tag ''`` will match untagged snapshots only.
 
-Multiple policies will be ORed together so as to be as inclusive as possible
-for keeping snapshots.
+When ``forget`` is run with a policy, restic loads the list of all snapshots,
+then groups these by host name and list of directories. The grouping options can
+be set with ``--group-by``, to e.g. group snapshots by only paths and tags use
+``--group-by paths,tags``. The policy is then applied to each group of snapshots
+separately. This is a safety feature to prevent accidental removal of unrelated
+backup sets.
 
-Additionally, you can restrict removing snapshots to those which have a
-particular hostname with the ``--host`` parameter, or tags with the
-``--tag`` option. When multiple tags are specified, only the snapshots
-which have all the tags are considered. For example, the following command
-removes all but the latest snapshot of all snapshots that have the tag ``foo``:
+Additionally, you can restrict the policy to only process snapshots which have a
+particular hostname with the ``--host`` parameter, or tags with the ``--tag``
+option. When multiple tags are specified, only the snapshots which have all the
+tags are considered. For example, the following command removes all but the
+latest snapshot of all snapshots that have the tag ``foo``:
 
 .. code-block:: console
 
@@ -243,21 +250,8 @@ the tag.
 
    $ restic forget --tag '' --keep-last 1
 
-All the ``--keep-*`` options above only count
-hours/days/weeks/months/years which have a snapshot, so those without a
-snapshot are ignored.
-
-For safety reasons, restic refuses to act on an "empty" policy. For example,
-if one were to specify ``--keep-last 0`` to forget *all* snapshots in the
-repository, restic will respond that no snapshots will be removed. To delete
-all snapshots, use ``--keep-last 1`` and then finally remove the last
-snapshot ID manually (by passing the ID to ``forget``).
-
-All snapshots are evaluated against all matching ``--keep-*`` counts. A
-single snapshot on 2017-09-30 (Sat) will count as a daily, weekly and monthly.
-
-Let's explain this with an example: Suppose you have only made a backup
-on each Sunday for 12 weeks:
+Let's look at a simple example: Suppose you have only made one backup every
+Sunday for 12 weeks:
 
 .. code-block:: console
 
@@ -280,8 +274,8 @@ on each Sunday for 12 weeks:
    ---------------------------------------------------------------
    12 snapshots
 
-Then ``forget --keep-daily 4`` will keep the last four snapshots for the last
-four Sundays, but remove the rest:
+Then ``forget --keep-daily 4`` will keep the last four snapshots, for the last
+four Sundays, and remove the other snapshots:
 
 .. code-block:: console
 
@@ -312,29 +306,86 @@ four Sundays, but remove the rest:
    ---------------------------------------------------------------
    8 snapshots
 
-The result of the ``forget --keep-daily`` operation does not depend on when it
-is run, it will only count the days for which a snapshot exists. This is a
-safety feature: it prevents restic from removing snapshots when no new ones are
-created. Otherwise, running ``forget --keep-daily 4`` on a Friday (without any
-snapshot Monday to Thursday) would remove all snapshots!
+The processed snapshots are evaluated against all ``--keep-*`` options but a
+snapshot only need to match a single option to be kept (the results are ORed).
+This means that the most recent snapshot on a Sunday would match both hourly,
+daily and weekly ``--keep-*`` options, and possibly more depending on calendar.
 
-Another example: Suppose you make daily backups for 100 years. Then
-``forget --keep-daily 7 --keep-weekly 5 --keep-monthly 12 --keep-yearly 75``
-will keep the most recent 7 daily snapshots, then 4 (remember, 7 dailies
-already include a week!) last-day-of-the-weeks and 11 or 12
-last-day-of-the-months (11 or 12 depends if the 5 weeklies cross a month).
-And finally 75 last-day-of-the-year snapshots. All other snapshots are
-removed.
+For example, suppose you make one backup every day for 100 years. Then ``forget
+--keep-daily 7 --keep-weekly 5 --keep-monthly 12 --keep-yearly 75`` would keep
+the most recent 7 daily snapshots and 4 last-day-of-the-week ones (since the 7
+dailies already include 1 weekly). Additionally, 12 or 11 last-day-of-the-month
+snapshots will be kept (depending on whether one of them ends up being the same
+as a daily or weekly). And finally 75 or 74 last-day-of-the-year snapshots are
+kept, depending on whether one of them ends up being the same as an already kept
+snapshot. All other snapshots are removed.
 
-You might want to maintain the same policy as for the example above, but have
+You might want to maintain the same policy as in the example above, but have
 irregular backups. For example, the 7 snapshots specified with ``--keep-daily 7`` 
-might be spread over a longer period. If what you want is to keep daily snapshots
-for a week, weekly for a month, monthly for a year and yearly for 75 years, you 
-could specify:
-``forget --keep-daily-within 7d --keep-weekly-within 1m --keep-monthly-within 1y
---keep-yearly-within 75y``
-(Note that `1w` is not a recognized duration, so you will have to specify 
-`7d` instead)
+might be spread over a longer period. If what you want is to keep daily
+snapshots for the last week, weekly for the last month, monthly for the last
+year and yearly for the last 75 years, you can instead specify ``forget
+--keep-within-daily 7d --keep-within-weekly 1m --keep-within-monthly 1y
+--keep-within-yearly 75y`` (note that `1w` is not a recognized duration, so
+you will have to specify `7d` instead).
+
+For safety reasons, restic refuses to act on an "empty" policy. For example,
+if one were to specify ``--keep-last 0`` to forget *all* snapshots in the
+repository, restic will respond that no snapshots will be removed. To delete
+all snapshots, use ``--keep-last 1`` and then finally remove the last snapshot
+manually (by passing the ID to ``forget``).
+
+Security considerations in append-only mode
+===========================================
+
+.. note:: TL;DR: With append-only repositories, one should specifically use the
+    ``--keep-within`` option of the ``forget`` command when removing snapshots.
+
+To prevent a compromised backup client from deleting its backups (for example
+due to a ransomware infection), a repository service/backend can serve the
+repository in a so-called append-only mode. This means that the repository is
+served in such a way that it can only be written to and read from, while delete
+and overwrite operations are denied. Restic's `rest-server`_ features an
+append-only mode, but few other standard backends do. To support append-only
+with such backends, one can use `rclone`_ as a complement in between the backup
+client and the backend service.
+
+.. _rest-server: https://github.com/restic/rest-server/
+.. _rclone: https://rclone.org/commands/rclone_serve_restic/
+
+To remove snapshots and recover the corresponding disk space, the ``forget``
+and ``prune`` commands require full read, write and delete access to the
+repository. If an attacker has this, the protection offered by append-only
+mode is naturally void. The usual and recommended setup with append-only
+repositories is therefore to use a separate and well-secured client whenever
+full access to the repository is needed, e.g. for administrative tasks such
+as running ``forget``, ``prune`` and other maintenance commands.
+
+However, even with append-only mode active and a separate, well-secured client
+used for administrative tasks, an attacker who is able to add garbage snapshots
+to the repository could bring the snapshot list into a state where all the
+legitimate snapshots risk being deleted by an unsuspecting administrator that
+runs the ``forget`` command with certain ``--keep-*`` options, leaving only the
+attacker's useless snapshots.
+
+For example, if the ``forget`` policy is to keep three weekly snapshots, and
+the attacker adds an empty snapshot for each of the last three weeks, all with
+a timestamp (see the ``backup`` command's ``--time`` option) slightly more
+recent than the existing snapshots (but still within the target week), then the
+next time the repository administrator (or a scheduled job) runs the ``forget``
+command with this policy, the legitimate snapshots will be removed (since the
+policy will keep only the most recent snapshot within each week). Even without
+running ``prune``, recovering data would be messy and some metadata lost.
+
+To avoid this, ``forget`` policies applied to append-only repositories should
+use the ``--keep-within`` option, as this will keep not only the attacker's
+snapshots but also the legitimate ones. Assuming the system time is correctly
+set when ``forget`` runs, this will allow the administrator to notice problems
+with the backup or the compromised host (e.g. by seeing more snapshots than
+usual or snapshots with suspicious timestamps). This is, of course, limited to
+the specified duration: if ``forget --keep-within 7d`` is run 8 days after the
+last good snapshot, then the attacker can still use that opportunity to remove
+all legitimate snapshots.
 
 Customize pruning
 *****************

doc/080_examples.rst

@@ -21,7 +21,7 @@ Setting up restic with Amazon S3
 Preface
 =======
 
-This tutorial will show you how to use restic with AWS S3. It will show you how
+This tutorial will show you how to use restic with Amazon S3. It will show you how
 to navigate the AWS web interface, create an S3 bucket, create a user with
 access to only this bucket, and finally how to connect restic to this bucket.
 
@@ -226,7 +226,7 @@ repository:
    the repository. Losing your password means that your data is
    irrecoverably lost.
 
-restic is now ready to be used with AWS S3. Try to create a backup:
+restic is now ready to be used with Amazon S3. Try to create a backup:
 
 .. code-block:: console
 
@@ -247,7 +247,7 @@ restic is now ready to be used with AWS S3. Try to create a backup:
    ----------------------------------------------------------------------
    10fdbace  2017-03-26 16:41:50  blackbox                /home/philip/restic-demo/test.bin
 
-A snapshot was created and stored in the S3 bucket. By default backups to AWS S3 will use the ``STANDARD`` storage class. Available storage classes include ``STANDARD``, ``STANDARD_IA``, ``ONEZONE_IA``, ``INTELLIGENT_TIERING``, and ``REDUCED_REDUNDANCY``. A different storage class could have been specified in the above command by using ``-o`` or ``--option``:
+A snapshot was created and stored in the S3 bucket. By default backups to Amazon S3 will use the ``STANDARD`` storage class. Available storage classes include ``STANDARD``, ``STANDARD_IA``, ``ONEZONE_IA``, ``INTELLIGENT_TIERING``, and ``REDUCED_REDUNDANCY``. A different storage class could have been specified in the above command by using ``-o`` or ``--option``:
 
 .. code-block:: console
 
@@ -309,7 +309,7 @@ the backups:
 
 .. code-block:: console
 
-   root@a3e580b6369d:/# useradd -m restic
+   root@a3e580b6369d:/# useradd --system --create-home --shell /sbin/nologin restic
 
 Then we download and install the restic binary into the user's home 
 directory (please adjust the URL to refer to the latest restic version).
@@ -317,7 +317,7 @@ directory (please adjust the URL to refer to the latest restic version).
 .. code-block:: console
 
    root@a3e580b6369d:/# mkdir ~restic/bin
-   root@a3e580b6369d:/# curl -L https://github.com/restic/restic/releases/download/v0.9.6/restic_0.9.6_linux_amd64.bz2 | bunzip2 > ~restic/bin/restic
+   root@a3e580b6369d:/# curl -L https://github.com/restic/restic/releases/download/v0.12.1/restic_0.12.1_linux_amd64.bz2 | bunzip2 > ~restic/bin/restic
 
 Before we assign any special capability to the restic binary we
 restrict its permissions so that only root and the newly created

doc/090_participating.rst

@@ -83,10 +83,9 @@ back end is contained in `Design <https://restic.readthedocs.io/en/latest/design
 If you'd like to start contributing to restic, but don't know exactly
 what do to, have a look at this great article by Dave Cheney:
 `Suggestions for contributing to an Open Source
-project <https://dave.cheney.net/2016/03/12/suggestions-for-contributing-to-an-open-source-project>`__
+project <https://dave.cheney.net/2016/03/12/suggestions-for-contributing-to-an-open-source-project>`__.
 A few issues have been tagged with the label ``help wanted``, you can
-start looking at those:
-https://github.com/restic/restic/labels/help%20wanted
+start looking at `those <https://github.com/restic/restic/labels/help%3A%20wanted>`_.
 
 ********
 Security

doc/bash-completion.sh

@@ -367,6 +367,10 @@ _restic_backup()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--dry-run")
+    flags+=("-n")
+    local_nonpersistent_flags+=("--dry-run")
+    local_nonpersistent_flags+=("-n")
     flags+=("--exclude=")
     two_word_flags+=("--exclude")
     two_word_flags+=("-e")
@@ -454,6 +458,7 @@ _restic_backup()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -519,6 +524,7 @@ _restic_cache()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -576,6 +582,7 @@ _restic_cat()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -643,6 +650,7 @@ _restic_check()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -734,6 +742,7 @@ _restic_copy()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -793,6 +802,7 @@ _restic_diff()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -870,6 +880,7 @@ _restic_dump()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -975,6 +986,7 @@ _restic_find()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1134,6 +1146,7 @@ _restic_forget()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1207,6 +1220,7 @@ _restic_generate()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1260,6 +1274,7 @@ _restic_help()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1340,6 +1355,7 @@ _restic_init()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1409,6 +1425,7 @@ _restic_key()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1466,6 +1483,7 @@ _restic_list()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1543,6 +1561,7 @@ _restic_ls()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1604,6 +1623,7 @@ _restic_migrate()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1685,6 +1705,7 @@ _restic_mount()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1756,6 +1777,7 @@ _restic_prune()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1815,6 +1837,7 @@ _restic_rebuild-index()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1872,6 +1895,7 @@ _restic_recover()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -1971,6 +1995,7 @@ _restic_restore()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -2032,6 +2057,7 @@ _restic_self-update()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -2117,6 +2143,7 @@ _restic_snapshots()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -2192,6 +2219,7 @@ _restic_stats()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -2275,6 +2303,7 @@ _restic_tag()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -2334,6 +2363,7 @@ _restic_unlock()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -2391,6 +2421,7 @@ _restic_version()
     flags+=("--cache-dir=")
     two_word_flags+=("--cache-dir")
     flags+=("--cleanup-cache")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")
@@ -2475,6 +2506,7 @@ _restic_root_command()
     flags+=("-h")
     local_nonpersistent_flags+=("--help")
     local_nonpersistent_flags+=("-h")
+    flags+=("--insecure-tls")
     flags+=("--json")
     flags+=("--key-hint=")
     two_word_flags+=("--key-hint")

doc/design.rst

@@ -125,7 +125,7 @@ s3 backend ``s3.layout``.
 S3 Legacy Layout
 ----------------
 
-Unfortunately during development the AWS S3 backend uses slightly different
+Unfortunately during development the Amazon S3 backend uses slightly different
 paths (directory names use singular instead of plural for ``key``,
 ``lock``, and ``snapshot`` files), and the pack files are stored directly below
 the ``data`` directory. The S3 Legacy repository layout looks like this:
@@ -607,7 +607,7 @@ examples of things an adversary could achieve in various circumstances.
 An adversary with read access to your backup storage location could:
 
 -  Attempt a brute force password guessing attack against a copy of the
-   repository (even more reason to use long, 30+ character passwords).
+   repository (please use strong passwords with sufficient entropy).
 -  Infer which packs probably contain trees via file access patterns.
 -  Infer the size of backups by using creation timestamps of repository objects.
 
@@ -618,7 +618,7 @@ An adversary with network access could:
 -  Determine from where you create your backups (i.e., the location where the
    requests originate).
 -  Determine where you store your backups (i.e., which provider/target system).
--  Infer the size of backups by using creation timestamps of repository objects.
+-  Infer the size of backups by observing network traffic.
 
 The following are examples of the implications associated with violating some
 of the aforementioned assumptions.
@@ -629,11 +629,11 @@ system making backups could:
 -  Render the entire backup process untrustworthy (e.g., intercept password, 
    copy files, manipulate data).
 -  Create snapshots (containing garbage data) which cover all modified files
-   and wait until a trusted host has used forget often enough to forget all
+   and wait until a trusted host has used ``forget`` often enough to remove all
    correct snapshots.
--  Create a garbage snapshot for every existing snapshot with a slightly different
-   timestamp and wait until forget has run, thereby removing all correct
-   snapshots at once.
+-  Create a garbage snapshot for every existing snapshot with a slightly
+   different timestamp and wait until certain ``forget`` configurations have been
+   run, thereby removing all correct snapshots at once.
 
 An adversary with write access to your files at the storage location could:
 
@@ -645,21 +645,27 @@ An adversary with write access to your files at the storage location could:
    the snapshot cannot be restored completely. Restic is not designed to detect
    this attack.
 
-An adversary who compromises a host system with append-only access to the 
-backup repository could:
+An adversary who compromises a host system with append-only (read+write allowed,
+delete+overwrite denied) access to the backup repository could:
 
+-  Capture the password and decrypt backups from the past and in the future
+   (see the "leaked key" example below for related information).
 -  Render new backups untrustworthy *after* the host has been compromised
    (due to having complete control over new backups). An attacker cannot delete
    or manipulate old backups. As such, restoring old snapshots created *before*
    a host compromise remains possible.
-   *Note: It is **not** recommended to ever run forget automatically for an
-   append-only backup to which a potentially compromised host has access
-   because an attacker using fake snapshots could cause forget to remove
-   correct snapshots.*
+-  Potentially manipulate the use of the ``forget`` command into deleting all
+   legitimate snapshots, keeping only bogus snapshots added by the attacker.
+   Ransomware might try this in order to leave only one option to get your data
+   back: paying the ransom. For safe use of ``forget``, please see the
+   corresponding documentation on removing backup snapshots and append-only mode.
 
-An adversary who has a leaked key for a repository which has not been re-encrypted
-could:
-
--  Decrypt existing and future backup data. If multiple hosts backup into the same
-   repository, an attacker will get access to the backup data of every host.
+An adversary who has a leaked (decrypted) key for a repository could:
 
+-  Decrypt existing and future backup data. If multiple hosts backup into the
+   same repository, an attacker will get access to the backup data of every host.
+   Note that since the local encryption key gives access to the master key, a
+   password change will not prevent this. Changing the master key can currently
+   only be done using the ``copy`` command, which moves the data into a new
+   repository with a new master key, or by making a completely new repository
+   and new backup.

doc/developer_information.rst

@@ -12,27 +12,28 @@ depends on the following things:
  * The source code for the release
  * The exact version of the official `Go compiler <https://golang.org>`__ used to produce the binaries (running ``restic version`` will print this)
  * The architecture and operating system the Go compiler runs on (Linux, ``amd64``)
+ * The build tags (for official binaries, it's the tag ``selfupdate``)
  * The path where the source code is extracted to (``/restic``)
  * The path to the Go compiler (``/usr/local/go``)
- * The build tags (for official binaries, it's the tag ``selfupdate``)
- * The environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``)
+ * The path to the Go workspace (``GOPATH=/home/build/go``)
+ * Other environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``)
 
 In addition, The compressed ZIP files for Windows depends on the modification
 timestamp and filename of the binary contained in it. In order to reproduce the
 exact same ZIP file every time, we update the timestamp of the file ``VERSION``
 in the source code archive and set the timezone to Europe/Berlin.
 
-In the following example, we'll use the file ``restic-0.10.0.tar.gz`` and Go
-1.15.2 to reproduce the released binaries.
+In the following example, we'll use the file ``restic-0.12.1.tar.gz`` and Go
+1.16.6 to reproduce the released binaries.
 
 1. Determine the Go compiler version used to build the released binaries, then download and extract the Go compiler into ``/usr/local/go``:
 
 .. code::
 
     $ restic version
-    restic 0.10.0 compiled with go1.15.2 on linux/amd64
+    restic 0.12.1 compiled with go1.16.6 on linux/amd64
     $ cd /usr/local
-    $ curl -L https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz | tar xz
+    $ curl -L https://dl.google.com/go/go1.16.6.linux-amd64.tar.gz | tar xz
 
 2. Extract the restic source code into ``/restic``
 
@@ -40,22 +41,23 @@ In the following example, we'll use the file ``restic-0.10.0.tar.gz`` and Go
 
     $ mkdir /restic
     $ cd /restic
-    $ TZ=Europe/Berlin curl -L https://github.com/restic/restic/releases/download/v0.10.0/restic-0.10.0.tar.gz | tar xz --strip-components=1
+    $ TZ=Europe/Berlin curl -L https://github.com/restic/restic/releases/download/v0.12.1/restic-0.12.1.tar.gz | tar xz --strip-components=1
 
 3. Build the binaries for Windows and Linux:
 
 .. code::
 
     $ export PATH=/usr/local/go/bin:$PATH
+    $ export GOPATH=/home/build/go
     $ go version
-    go version go1.15.2 linux/amd64
+    go version go1.16.6 linux/amd64
 
     $ GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w" -tags selfupdate -o restic_linux_amd64 ./cmd/restic
     $ bzip2 restic_linux_amd64
 
-    $ GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w" -tags selfupdate -o restic_0.10.0_windows_amd64.exe ./cmd/restic
-    $ touch --reference VERSION restic_0.10.0_windows_amd64.exe
-    $ TZ=Europe/Berlin zip -q -X restic_0.10.0_windows_amd64.zip restic_0.10.0_windows_amd64.exe
+    $ GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w" -tags selfupdate -o restic_0.12.1_windows_amd64.exe ./cmd/restic
+    $ touch --reference VERSION restic_0.12.1_windows_amd64.exe
+    $ TZ=Europe/Berlin zip -q -X restic_0.12.1_windows_amd64.zip restic_0.12.1_windows_amd64.exe
 
 Building the Official Binaries
 ******************************
@@ -83,7 +85,7 @@ The following steps are necessary to build the binaries:
 
 .. code::
 
-    tar xvzf restic-0.10.0.tar.gz
+    tar xvzf restic-0.12.1.tar.gz
 
 3. Create a directory to place the resulting binaries in:
 
@@ -96,20 +98,20 @@ The following steps are necessary to build the binaries:
 .. code::
 
     docker run --rm \
-        --volume "$PWD/restic-0.10.0:/restic" \
+        --volume "$PWD/restic-0.12.1:/restic" \
         --volume "$PWD/output:/output" \
         restic/builder \
-        go run helpers/build-release-binaries/main.go --version 0.10.0
+        go run helpers/build-release-binaries/main.go --version 0.12.1
 
 4. If anything goes wrong, you can enable debug output like this:
 
 .. code::
 
     docker run --rm \
-        --volume "$PWD/restic-0.10.0:/restic" \
+        --volume "$PWD/restic-0.12.1:/restic" \
         --volume "$PWD/output:/output" \
         restic/builder \
-        go run helpers/build-release-binaries/main.go --version 0.10.0 --verbose
+        go run helpers/build-release-binaries/main.go --version 0.12.1 --verbose
 
 Prepare a New Release
 *********************
@@ -122,6 +124,6 @@ required argument is the new version number (in `Semantic Versioning
 
 .. code::
 
-    go run helpers/prepare-release/main.go 0.10.0
+    go run helpers/prepare-release/main.go 0.12.1
 
 Checks can be skipped on demand via flags, please see ``--help`` for details.

doc/faq.rst

@@ -40,18 +40,21 @@ looks like this:
 ::
 
     $ restic check
-    Create exclusive lock for repository
-    Load indexes
-    Check all packs
+    create exclusive lock for repository
+    load indexes
+    check all packs
     pack 819a9a52e4f51230afa89aefbf90df37fb70996337ae57e6f7a822959206a85e: not referenced in any index
     pack de299e69fb075354a3775b6b045d152387201f1cdc229c31d1caa34c3b340141: not referenced in any index
-    Check snapshots, trees and blobs
-    Fatal: repository contains errors
+    2 additional files were found in the repo, which likely contain duplicate data.
+    You can run `restic prune` to correct this.
+    check snapshots, trees and blobs
+    [0:00] 100.00%  16 / 16 snapshots
+    no errors were found
 
 The message means that there is more data stored in the repo than
-strictly necessary. With high probability this is duplicate data. In
-order to clean it up, the command ``restic prune`` can be used. The
-cause of this bug is not yet known.
+strictly necessary. This is uncritical. With high probability this is duplicate data
+caused by an interrupted backup run or upload operation. In
+order to clean it up, the command ``restic prune`` can be used.
 
 I ran a ``restic`` command but it is not working as intended, what do I do now?
 -------------------------------------------------------------------------------
@@ -207,3 +210,21 @@ temporarily disable your antivirus software to find out if it is the cause for
 your performance problems. If you are certain that the antivirus software is
 the cause for this and you want to gain maximum performance, you have to add
 the restic binary to an exclusions list within the antivirus software.
+
+How do I choose a strong password?
+----------------------------------
+
+Length is the single most important component in password strength.  That doesn't
+mean that other components such as complexity and entropy (or randomness) are not
+important to consider.  A strong password includes Alphabetical, Numerical and
+Special characters. For example, ``nk3E9Rr26md6GGySyyWMrfakw8Jck4$&vVY6`` would
+be a very strong password, if not for being in this documentation.
+
+There are plenty of tools out there, such as OpenSSL, pwgen or KeePass that can
+generate a sufficiently complex, random and long password.
+
+Restic backup command fails to find a valid file in Windows
+-----------------------------------------------------------
+
+If the name of a file in Windows contains an invalid character, Restic will not be
+able to read the file. To solve this issue, consider renaming the particular file.

doc/man/restic-backup.1

@@ -25,6 +25,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 
 
 .SH OPTIONS
+.PP
+\fB\-n\fP, \fB\-\-dry\-run\fP[=false]
+	do not upload or write any data, just show what would be done
+
 .PP
 \fB\-e\fP, \fB\-\-exclude\fP=[]
 	exclude a \fB\fCpattern\fR (can be specified multiple times)
@@ -91,7 +95,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 
 .PP
 \fB\-\-parent\fP=""
-	use this parent \fB\fCsnapshot\fR (default: last snapshot in the repo that has the same target files/directories)
+	use this parent \fB\fCsnapshot\fR (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)
 
 .PP
 \fB\-\-stdin\fP[=false]
@@ -127,6 +131,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-cache.1

@@ -52,6 +52,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-cat.1

@@ -40,6 +40,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-check.1

@@ -41,7 +41,7 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 
 .PP
 \fB\-\-read\-data\-subset\fP=""
-	read a \fB\fCsubset\fR of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset
+	read a \fB\fCsubset\fR of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset
 
 .PP
 \fB\-\-with\-cache\fP[=false]
@@ -61,6 +61,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-copy.1

@@ -81,6 +81,10 @@ new destination repository using the "init" command.
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-diff.1

@@ -60,6 +60,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-dump.1

@@ -63,6 +63,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-find.1

@@ -85,6 +85,10 @@ It can also be used to search for restic blobs or trees for troubleshooting.
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-forget.1

@@ -15,8 +15,9 @@ restic\-forget \- Remove snapshots from the repository
 .PP
 The "forget" command removes snapshots according to a policy. Please note that
 this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove this (now unreferenced)
-data after 'forget' was run successfully, see the 'prune' command.
+is a reference to data stored there. In order to remove the unreferenced data
+after "forget" was run successfully, see the "prune" command. Please also read
+the documentation for "forget" to learn about important security considerations.
 
 
 .SH EXIT STATUS
@@ -135,6 +136,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-generate.1

@@ -57,6 +57,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-init.1

@@ -64,6 +64,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-key.1

@@ -52,6 +52,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-list.1

@@ -40,6 +40,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-ls.1

@@ -43,7 +43,7 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 
 .PP
 \fB\-H\fP, \fB\-\-host\fP=[]
-	only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times)
+	only consider snapshots for this \fB\fChost\fR, when snapshot ID "latest" is given (can be specified multiple times)
 
 .PP
 \fB\-l\fP, \fB\-\-long\fP[=false]
@@ -51,7 +51,7 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 
 .PP
 \fB\-\-path\fP=[]
-	only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot ID is given
+	only consider snapshots which include this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times)
 
 .PP
 \fB\-\-recursive\fP[=false]
@@ -59,7 +59,7 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 
 .PP
 \fB\-\-tag\fP=[]
-	only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot ID is given
+	only consider snapshots which include this \fB\fCtaglist\fR, when snapshot ID "latest" is given (can be specified multiple times)
 
 
 .SH OPTIONS INHERITED FROM PARENT COMMANDS
@@ -75,6 +75,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-migrate.1

@@ -45,6 +45,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-mount.1

@@ -100,6 +100,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-prune.1

@@ -57,6 +57,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it

doc/man/restic-rebuild-index.1

@@ -45,6 +45,10 @@ Exit status is 0 if the command was successful, and non\-zero if there was any e
 \fB\-\-cleanup\-cache\fP[=false]
 	auto remove old cache directories
 
+.PP
+\fB\-\-insecure\-tls\fP[=false]
+	skip TLS certificate verification when connecting to the repo (insecure)
+
 .PP
 \fB\-\-json\fP[=false]
 	set output mode to JSON for commands that support it