report result of delete operation after failed upload

s3: Fix sanity check
The sanity check shouldn't replace the error message if there is already one.
2026-10-30 12:05:42 +00:00 · 2021-03-08 21:54:46 +01:00 · 2021-03-08 21:54:46 +01:00 · 2021-03-08 21:54:46 +01:00
279 changed files with 3424 additions and 9737 deletions
@@ -1,12 +0,0 @@
 # Folders
 .git/
 .github/
 changelog/
 doc/
 docker/
 helpers/
 # Files
 .gitignore
 .golangci.yml
 *.md
@@ -1,7 +1,13 @@
 <!--
 Thank you very much for contributing code or documentation to restic! Please
 fill out the following questions to make it easier for us to review your
 changes.
 You do not need to check all the boxes below all at once, feel free to take
 your time and add more commits. If you're done and ready for review, please
 check the last box.
 -->
 What does this PR change? What problem does it solve?
@@ -11,8 +17,8 @@ What does this PR change? What problem does it solve?
 Describe the changes and their purpose here, as detailed as needed.
 -->
-Was the change previously discussed in an issue or on the forum?
+Was the change discussed in an issue or in the forum before?
----------------------------------------------------------------
+------------------------------------------------------------
 <!--
 Link issues and relevant forum posts here.
@@ -24,17 +30,11 @@ is closed automatically when this PR is merged.
 Checklist
 ---------
-<!--
+- [ ] I have read the [Contribution Guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches)
-You do not need to check all the boxes below all at once. Feel free to take
+- [ ] I have enabled [maintainer edits for this PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork)
-your time and add more commits. If you're done and ready for review, please
+- [ ] I have added tests for all changes in this PR
-check the last box. Enable a checkbox by replacing [ ] with [x].
+- [ ] I have added documentation for the changes (in the manual)
-->
+- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/TEMPLATE))
-
+- [ ] I have run `gofmt` on the code in all commits
- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
+- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits)
- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
+- [ ] I'm done, this Pull Request is ready for review
 - [ ] I have added tests for all code changes.
 - [ ] I have added documentation for relevant changes (in the manual).
 - [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
 - [ ] I have run `gofmt` on the code in all commits.
 - [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
 - [ ] I'm done! This pull request is ready for review.
@@ -8,10 +8,6 @@ on:
  # run tests for all pull requests
  pull_request:
 env:
  latest_go: "1.18.x"
  GO111MODULE: on
 jobs:
  test:
    strategy:
@@ -19,47 +15,30 @@ jobs:
        # list of jobs to run:
        include:
          - job_name: Windows
-            go: 1.18.x
+            go: 1.15.x
            os: windows-latest
            install_verb: install
          - job_name: macOS
-            go: 1.18.x
+            go: 1.15.x
            os: macOS-latest
            test_fuse: false
            install_verb: install
          - job_name: Linux
            go: 1.18.x
            os: ubuntu-latest
            test_cloud_backends: true
            test_fuse: true
            check_changelog: true
            install_verb: install
          - job_name: Linux
            go: 1.17.x
            os: ubuntu-latest
            test_fuse: true
            install_verb: install
          - job_name: Linux
            go: 1.16.x
            os: ubuntu-latest
            test_fuse: true
            install_verb: get
          - job_name: Linux
            go: 1.15.x
            os: ubuntu-latest
            test_cloud_backends: true
            test_fuse: true
-            install_verb: get
+            check_changelog: true
          - job_name: Linux
            go: 1.14.x
            os: ubuntu-latest
            test_fuse: true
-            install_verb: get
+
          - job_name: Linux
            go: 1.13.x
            os: ubuntu-latest
            test_fuse: true
    name: ${{ matrix.job_name }} Go ${{ matrix.go }}
    runs-on: ${{ matrix.os }}
@@ -76,7 +55,7 @@ jobs:
      - name: Get programs (Linux/macOS)
        run: |
          echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
+          go get github.com/restic/rest-server/...
          echo "install minio server"
          mkdir $HOME/bin
@@ -108,7 +87,7 @@ jobs:
          $ProgressPreference = 'SilentlyContinue'
          echo "build Go tools"
-          go ${{ matrix.install_verb }} github.com/restic/rest-server/...
+          go get github.com/restic/rest-server/...
          echo "install minio server"
          mkdir $Env:USERPROFILE/bin
@@ -198,7 +177,7 @@ jobs:
      - name: Check changelog files with calens
        run: |
          echo "install calens"
-          go install github.com/restic/calens@latest
+          go get github.com/restic/calens
          echo "check changelog files"
          calens
@@ -211,17 +190,18 @@ jobs:
      matrix:
        # run cross-compile in two batches parallel so the overall tests run faster
        targets:
-          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
+          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le \
            openbsd/386 openbsd/amd64"
          - "freebsd/386 freebsd/amd64 freebsd/arm \
            aix/ppc64 \
-            darwin/amd64 darwin/arm64 \
+            darwin/amd64 \
            netbsd/386 netbsd/amd64 \
            windows/386 windows/amd64 \
            solaris/amd64"
    env:
      go: 1.15.x
      GOPROXY: https://proxy.golang.org
    runs-on: ubuntu-latest
@@ -229,18 +209,18 @@ jobs:
    name: Cross Compile for ${{ matrix.targets }}
    steps:
-      - name: Set up Go ${{ env.latest_go }}
+      - name: Set up Go ${{ env.go }}
        uses: actions/setup-go@v2
        with:
-          go-version: ${{ env.latest_go }}
+          go-version: ${{ env.go }}
      - name: Install gox
        run: |
          go install github.com/mitchellh/gox@latest
      - name: Check out code
        uses: actions/checkout@v2
      - name: Install gox
        run: |
          go get github.com/mitchellh/gox
      - name: Cross-compile with gox for ${{ matrix.targets }}
        env:
          GOFLAGS: "-trimpath"
@@ -254,11 +234,6 @@ jobs:
    name: lint
    runs-on: ubuntu-latest
    steps:
      - name: Set up Go ${{ env.latest_go }}
        uses: actions/setup-go@v2
        with:
          go-version: ${{ env.latest_go }}
      - name: Check out code
        uses: actions/checkout@v2
@@ -266,11 +241,10 @@ jobs:
        uses: golangci/golangci-lint-action@v2
        with:
          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.45
+          version: v1.36
          # Optional: show only new issues if it's a pull request. The default value is `false`.
          only-new-issues: true
          args: --verbose --timeout 5m
          skip-go-installation: true
        # only run golangci-lint for pull requests, otherwise ALL hints get
        # reported. We need to slowly address all issues until we can enable
@@ -282,44 +256,3 @@ jobs:
          echo "check if go.mod and go.sum are up to date"
          go mod tidy
          git diff --exit-code go.mod go.sum
  docker:
    name: docker
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v2
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v3
        with:
          # list of Docker images to use as base name for tags
          images: |
            restic/restic
          # generate Docker tags based on the following events/attributes
          tags: |
            type=schedule
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}
            type=sha
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v2
        with:
          push: false
          context: .
          file: docker/Dockerfile
          pull: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
@@ -1,654 +1,3 @@
 Changelog for restic 0.13.0 (2022-03-26)
 =======================================
 The following sections list the changes in restic 0.13.0 relevant to
 restic users. The changes are ordered by importance.
 Summary
 -------
 * Fix #1106: Never lock repository for `list locks`
 * Fix #2345: Make cache crash-resistant and usable by multiple concurrent processes
 * Fix #2452: Improve error handling of repository locking
 * Fix #2738: Don't print progress for `backup --json --quiet`
 * Fix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
 * Fix #3518: Make `copy` command honor `--no-lock` for source repository
 * Fix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
 * Fix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
 * Fix #3667: The `mount` command now reports symlinks sizes
 * Fix #3488: `rebuild-index` failed if an index file was damaged
 * Fix #3591: Fix handling of `prune --max-repack-size=0`
 * Fix #3619: Avoid choosing parent snapshots newer than time of new snapshot
 * Chg #3641: Ignore parent snapshot for `backup --stdin`
 * Chg #3519: Require Go 1.14 or newer
 * Enh #1542: Add `--dry-run`/`-n` option to `backup` command
 * Enh #2202: Add upload checksum for Azure, GS, S3 and Swift backends
 * Enh #233: Support negative include/exclude patterns
 * Enh #2388: Add warning for S3 if partial credentials are provided
 * Enh #2508: Support JSON output and quiet mode for the `diff` command
 * Enh #2656: Add flag to disable TLS verification for self-signed certificates
 * Enh #3003: Atomic uploads for the SFTP backend
 * Enh #3127: Add xattr (extended attributes) support for Solaris
 * Enh #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
 * Enh #3490: Support random subset by size in `check --read-data-subset`
 * Enh #3541: Improve handling of temporary B2 delete errors
 * Enh #3542: Add file mode in symbolic notation to `ls --json`
 * Enh #2594: Speed up the `restore --verify` command
 * Enh #2816: The `backup` command no longer updates file access times on Linux
 * Enh #2880: Make `recover` collect only unreferenced trees
 * Enh #3429: Verify that new or modified keys are stored correctly
 * Enh #3436: Improve local backend's resilience to (system) crashes
 * Enh #3508: Cache blobs read by the `dump` command
 * Enh #3511: Support configurable timeout for the rclone backend
 * Enh #3593: Improve `copy` performance by parallelizing IO
 Details
 -------
 * Bugfix #1106: Never lock repository for `list locks`
   The `list locks` command previously locked to the repository by default. This had the problem
   that it wouldn't work for an exclusively locked repository and that the command would also
   display its own lock file which can be confusing.
   Now, the `list locks` command never locks the repository.
   https://github.com/restic/restic/issues/1106
   https://github.com/restic/restic/pull/3665
 * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes
   The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of
   restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a
   temporary location before being moved to their proper location.
   This also allows multiple concurrent restic processes to operate on a single repository
   without conflicts. Previously, concurrent operations could cause segfaults because the
   processes saw each other's partially downloaded files.
   https://github.com/restic/restic/issues/2345
   https://github.com/restic/restic/pull/2838
 * Bugfix #2452: Improve error handling of repository locking
   Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly
   created one. Instead it continued trying to delete the old (usually no longer existing) lock
   file and thus over time lots of lock files accumulated. This has now been fixed.
   https://github.com/restic/restic/issues/2452
   https://github.com/restic/restic/issues/2473
   https://github.com/restic/restic/issues/2562
   https://github.com/restic/restic/pull/3512
 * Bugfix #2738: Don't print progress for `backup --json --quiet`
   Unlike the text output, the `--json` output format still printed progress information even in
   `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode.
   https://github.com/restic/restic/issues/2738
   https://github.com/restic/restic/pull/3264
 * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
   Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable,
   which caused problems in certain system/usage configurations. This has now been fixed.
   https://github.com/restic/restic/issues/3382
   https://github.com/restic/restic/pull/3474
 * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository
   The `copy` command previously did not respect the `--no-lock` option for the source
   repository, causing failures with read-only storage backends. This has now been fixed such
   that the option is now respected.
   https://github.com/restic/restic/issues/3518
   https://github.com/restic/restic/pull/3589
 * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
   Previously, if a request failed with an SSL unknown certificate authority error, the B2
   backend retried indefinitely and restic would appear to hang.
   This has now been fixed and restic instead fails with an error message.
   https://github.com/restic/restic/issues/3556
   https://github.com/restic/restic/issues/2355
   https://github.com/restic/restic/pull/3571
 * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
   Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the
   backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time
   as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone
   stdio connection already closed" errors.
   This has now been fixed by restic starting the rclone process detached from the console restic
   runs in (similar to starting processes in a new process group on Linux), which enables restic to
   gracefully clean up rclone (which now never gets the `SIGINT`).
   https://github.com/restic/restic/issues/3601
   https://github.com/restic/restic/pull/3602
 * Bugfix #3667: The `mount` command now reports symlinks sizes
   Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They
   now have a size equal to the byte length of their target path, as required by POSIX.
   https://github.com/restic/restic/issues/3667
   https://github.com/restic/restic/pull/3668
 * Bugfix #3488: `rebuild-index` failed if an index file was damaged
   Previously, the `rebuild-index` command would fail with an error if an index file was damaged
   or truncated. This has now been fixed.
   On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or
   to manually delete the damaged index.
   https://github.com/restic/restic/pull/3488
 * Bugfix #3591: Fix handling of `prune --max-repack-size=0`
   Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been
   fixed.
   As a workaround, `--max-repack-size=1` can be used with older versions of restic.
   https://github.com/restic/restic/pull/3591
 * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot
   The `backup` command, when a `--parent` was not provided, previously chose the most recent
   matching snapshot as the parent snapshot. However, this didn't make sense when the user passed
   `--time` to create a new snapshot older than the most recent snapshot.
   Instead, `backup` now chooses the most recent snapshot which is not newer than the
   snapshot-being-created's timestamp, to avoid any time travel.
   https://github.com/restic/restic/pull/3619
 * Change #3641: Ignore parent snapshot for `backup --stdin`
   Restic uses a parent snapshot to speed up directory scanning when performing backups, but this
   only wasted time and memory when the backup source is stdin (using the `--stdin` option of the
   `backup` command), since no directory scanning is performed in this case.
   Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to
   skip some startup operations and saves a bit of resources.
   The `--parent` option is still available for `backup --stdin`, but is now ignored.
   https://github.com/restic/restic/issues/3641
   https://github.com/restic/restic/pull/3645
 * Change #3519: Require Go 1.14 or newer
   Restic now requires Go 1.14 to build. This allows it to use new standard library features
   instead of an external dependency.
   https://github.com/restic/restic/issues/3519
 * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command
   Testing exclude filters and other configuration options was error prone as wrong filters
   could cause files to be uploaded unintentionally. It was also not possible to estimate
   beforehand how much data would be uploaded.
   The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of
   a backup without actually writing anything to the repository.
   Passing -vv will log information about files that would be added, allowing for verification of
   source and exclusion options before running the real backup.
   https://github.com/restic/restic/issues/1542
   https://github.com/restic/restic/pull/2308
   https://github.com/restic/restic/pull/3210
   https://github.com/restic/restic/pull/3300
 * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends
   Previously only the B2 and partially the Swift backends verified the integrity of uploaded
   (encrypted) files. The verification works by informing the backend about the expected hash of
   the uploaded file. The backend then verifies the upload and thereby rules out any data
   corruption during upload.
   We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides
   integrity checking for uploads also means that restic can now be used to store backups in S3
   buckets which have Object Lock enabled.
   https://github.com/restic/restic/issues/2202
   https://github.com/restic/restic/issues/2700
   https://github.com/restic/restic/issues/3023
   https://github.com/restic/restic/pull/3246
 * Enhancement #233: Support negative include/exclude patterns
   If a pattern starts with an exclamation mark and it matches a file that was previously matched by
   a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to
   cancel the exclusion of some files.
   It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it
   is not possible to include files inside the directory.
   Example of use as an exclude pattern for the `backup` command:
   $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...]
   node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...]
   https://github.com/restic/restic/issues/233
   https://github.com/restic/restic/pull/2311
 * Enhancement #2388: Add warning for S3 if partial credentials are provided
   Previously restic did not notify about incomplete credentials when using the S3 backend,
   instead just reporting access denied.
   Restic now checks that both the AWS key ID and secret environment variables are set before
   connecting to the remote server, and reports an error if not.
   https://github.com/restic/restic/issues/2388
   https://github.com/restic/restic/pull/3532
 * Enhancement #2508: Support JSON output and quiet mode for the `diff` command
   The `diff` command now supports outputting machine-readable output in JSON format. To enable
   this, pass the `--json` option to the command. To only print the summary and suppress detailed
   output, pass the `--quiet` option.
   https://github.com/restic/restic/issues/2508
   https://github.com/restic/restic/pull/3592
 * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates
   There is now an `--insecure-tls` global option in restic, which disables TLS verification for
   self-signed certificates in order to support some development workflows.
   https://github.com/restic/restic/issues/2656
   https://github.com/restic/restic/pull/2657
 * Enhancement #3003: Atomic uploads for the SFTP backend
   The SFTP backend did not upload files atomically. An interrupted upload could leave an
   incomplete file behind which could prevent restic from accessing the repository. This has now
   been fixed and uploads in the SFTP backend are done atomically.
   https://github.com/restic/restic/issues/3003
   https://github.com/restic/restic/pull/3524
 * Enhancement #3127: Add xattr (extended attributes) support for Solaris
   Restic now supports xattr for the Solaris operating system.
   https://github.com/restic/restic/issues/3127
   https://github.com/restic/restic/pull/3628
 * Enhancement #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
   Restic used to silently ignore the `--no-lock` option of the `forget` command.
   It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If
   `--no-lock` option is specified without `--dry-run`, restic prints a warning message to
   stderr.
   https://github.com/restic/restic/issues/3464
   https://github.com/restic/restic/pull/3623
 * Enhancement #3490: Support random subset by size in `check --read-data-subset`
   The `--read-data-subset` option of the `check` command now supports a third way of specifying
   the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or
   t/T.
   https://github.com/restic/restic/issues/3490
   https://github.com/restic/restic/pull/3548
 * Enhancement #3541: Improve handling of temporary B2 delete errors
   Deleting files on B2 could sometimes fail temporarily, which required restic to retry the
   delete operation. In some cases the file was deleted nevertheless, causing the retries and
   ultimately the restic command to fail. This has now been fixed.
   https://github.com/restic/restic/issues/3541
   https://github.com/restic/restic/pull/3544
 * Enhancement #3542: Add file mode in symbolic notation to `ls --json`
   The `ls --json` command now provides the file mode in symbolic notation (using the
   `permissions` key), aligned with `find --json`.
   https://github.com/restic/restic/issues/3542
   https://github.com/restic/restic/pull/3573
   https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
 * Enhancement #2594: Speed up the `restore --verify` command
   The `--verify` option lets the `restore` command verify the file content after it has restored
   a snapshot. The performance of this operation has now been improved by up to a factor of two.
   https://github.com/restic/restic/pull/2594
 * Enhancement #2816: The `backup` command no longer updates file access times on Linux
   When reading files during backup, restic used to cause the operating system to update the
   files' access times. Note that this did not apply to filesystems with disabled file access
   times.
   Restic now instructs the operating system not to update the file access time, if the user
   running restic is the file owner or has root permissions.
   https://github.com/restic/restic/pull/2816
 * Enhancement #2880: Make `recover` collect only unreferenced trees
   Previously, the `recover` command used to generate a snapshot containing *all* root trees,
   even those which were already referenced by a snapshot.
   This has been improved such that it now only processes trees not already referenced by any
   snapshot.
   https://github.com/restic/restic/pull/2880
 * Enhancement #3429: Verify that new or modified keys are stored correctly
   When adding a new key or changing the password of a key, restic used to just create the new key (and
   remove the old one, when changing the password). There was no verification that the new key was
   stored correctly and works properly. As the repository cannot be decrypted without a valid key
   file, this could in rare cases cause the repository to become inaccessible.
   Restic now checks that new key files actually work before continuing. This can protect against
   some (rare) cases of hardware or storage problems.
   https://github.com/restic/restic/pull/3429
 * Enhancement #3436: Improve local backend's resilience to (system) crashes
   Restic now ensures that files stored using the `local` backend are created atomically (that
   is, files are either stored completely or not at all). This ensures that no incomplete files are
   left behind even if restic is terminated while writing a file.
   In addition, restic now tries to ensure that the directory in the repository which contains a
   newly uploaded file is also written to disk. This can prevent missing files if the system
   crashes or the disk is not properly unmounted.
   https://github.com/restic/restic/pull/3436
 * Enhancement #3508: Cache blobs read by the `dump` command
   When dumping a file using the `dump` command, restic did not cache blobs in any way, so even
   consecutive runs of the same blob were loaded from the repository again and again, slowing down
   the dump.
   Now, the caching mechanism already used by the `fuse` command is also used by the `dump`
   command. This makes dumping much faster, especially for sparse files.
   https://github.com/restic/restic/pull/3508
 * Enhancement #3511: Support configurable timeout for the rclone backend
   A slow rclone backend could cause restic to time out while waiting for the repository to open.
   Restic now offers an `-o rclone.timeout` option to make this timeout configurable.
   https://github.com/restic/restic/issues/3511
   https://github.com/restic/restic/pull/3514
 * Enhancement #3593: Improve `copy` performance by parallelizing IO
   Restic copy previously only used a single thread for copying blobs between repositories,
   which resulted in limited performance when copying small blobs to/from a high latency backend
   (i.e. any remote backend, especially b2).
   Copying will now use 8 parallel threads to increase the throughput of the copy operation.
   https://github.com/restic/restic/pull/3593
 Changelog for restic 0.12.1 (2021-08-03)
 =======================================
 The following sections list the changes in restic 0.12.1 relevant to
 restic users. The changes are ordered by importance.
 Summary
 -------
 * Fix #2742: Improve error handling for rclone and REST backend over HTTP2
 * Fix #3111: Fix terminal output redirection for PowerShell
 * Fix #3214: Treat an empty password as a fatal error for repository init
 * Fix #3267: `copy` failed to copy snapshots in rare cases
 * Fix #3184: `backup --quiet` no longer prints status information
 * Fix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
 * Fix #3302: Fix `fdopendir: not a directory` error for local backend
 * Fix #3334: Print `created new cache` message only on a terminal
 * Fix #3380: Fix crash of `backup --exclude='**'`
 * Fix #3305: Fix possibly missing backup summary of JSON output in case of error
 * Fix #3439: Correctly handle download errors during `restore`
 * Chg #3247: Empty files now have size of 0 in `ls --json` output
 * Enh #2780: Add release binaries for s390x architecture on Linux
 * Enh #3293: Add `--repository-file2` option to `init` and `copy` command
 * Enh #3312: Add auto-completion support for fish
 * Enh #3336: SFTP backend now checks for disk space
 * Enh #3377: Add release binaries for Apple Silicon
 * Enh #3414: Add `--keep-within-hourly` option to restic forget
 * Enh #3456: Support filtering and specifying untagged snapshots
 * Enh #3167: Allow specifying limit of `snapshots` list
 * Enh #3426: Optimize read performance of mount command
 * Enh #3427: `find --pack` fallback to index if data file is missing
 Details
 -------
 * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2
   When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect
   when no data was returned at all. This could cause for example the `check` command to report the
   following error:
   Pack ID does not match, want [...], got e3b0c442
   This has been fixed by correctly detecting and retrying the incomplete download.
   https://github.com/restic/restic/issues/2742
   https://github.com/restic/restic/pull/3453
   https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
 * Bugfix #3111: Fix terminal output redirection for PowerShell
   When redirecting the output of restic using PowerShell on Windows, the output contained
   terminal escape characters. This has been fixed by properly detecting the terminal type.
   In addition, the mintty terminal now shows progress output for the backup command.
   https://github.com/restic/restic/issues/3111
   https://github.com/restic/restic/pull/3325
 * Bugfix #3214: Treat an empty password as a fatal error for repository init
   When attempting to initialize a new repository, if an empty password was supplied, the
   repository would be created but the init command would return an error with a stack trace. Now,
   if an empty password is provided, it is treated as a fatal error, and no repository is created.
   https://github.com/restic/restic/issues/3214
   https://github.com/restic/restic/pull/3283
 * Bugfix #3267: `copy` failed to copy snapshots in rare cases
   The `copy` command could in rare cases fail with the error message `SaveTree(...) returned
   unexpected id ...`. This has been fixed.
   On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths.
   Note that, due to limitations in the repository format, these are not stored properly and
   should be avoided if possible.
   https://github.com/restic/restic/issues/3267
   https://github.com/restic/restic/pull/3310
 * Bugfix #3184: `backup --quiet` no longer prints status information
   A regression in the latest restic version caused the output of `backup --quiet` to contain
   large amounts of backup progress information when run using an interactive terminal. This is
   fixed now.
   A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`.
   https://github.com/restic/restic/issues/3184
   https://github.com/restic/restic/pull/3186
 * Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
   The command `restic check --read-data-subset=x%` crashed when run for an empty repository.
   This has been fixed.
   https://github.com/restic/restic/issues/3296
   https://github.com/restic/restic/pull/3309
 * Bugfix #3302: Fix `fdopendir: not a directory` error for local backend
   The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend
   when the `data` folder in the repository contained files. This has been fixed.
   https://github.com/restic/restic/issues/3302
   https://github.com/restic/restic/pull/3308
 * Bugfix #3334: Print `created new cache` message only on a terminal
   The message `created new cache` was printed even when the output wasn't a terminal. That broke
   piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now
   only printed on a terminal.
   https://github.com/restic/restic/issues/3334
   https://github.com/restic/restic/pull/3343
 * Bugfix #3380: Fix crash of `backup --exclude='**'`
   The exclude filter `**`, which excludes all files, caused restic to crash. This has been
   corrected.
   https://github.com/restic/restic/issues/3380
   https://github.com/restic/restic/pull/3393
 * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error
   When using `--json` output it happened from time to time that the summary output was missing in
   case an error occurred. This has been fixed.
   https://github.com/restic/restic/pull/3305
 * Bugfix #3439: Correctly handle download errors during `restore`
   Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download
   errors and only printed a warning. This has been fixed by retrying incomplete data downloads.
   https://github.com/restic/restic/issues/3439
   https://github.com/restic/restic/pull/3449
 * Change #3247: Empty files now have size of 0 in `ls --json` output
   The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size
   of zero explicitly for regular files, while omitting the size field for all other types.
   https://github.com/restic/restic/issues/3247
   https://github.com/restic/restic/pull/3257
 * Enhancement #2780: Add release binaries for s390x architecture on Linux
   We've added release binaries for Linux using the s390x architecture.
   https://github.com/restic/restic/issues/2780
   https://github.com/restic/restic/pull/3452
 * Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command
   The `init` and `copy` command can now be used with the `--repository-file2` option or the
   `$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the
   `--repo2` flag and allow you to read the destination repository from a file.
   Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or
   `init` command. The handling of this combination of options has been fixed. A workaround for
   this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`.
   https://github.com/restic/restic/issues/3293
   https://github.com/restic/restic/pull/3294
 * Enhancement #3312: Add auto-completion support for fish
   The `generate` command now supports fish auto completion.
   https://github.com/restic/restic/pull/3312
 * Enhancement #3336: SFTP backend now checks for disk space
   Backing up over SFTP previously spewed multiple generic "failure" messages when the remote
   disk was full. It now checks for disk space before writing a file and fails immediately with a "no
   space left on device" message.
   https://github.com/restic/restic/issues/3336
   https://github.com/restic/restic/pull/3345
 * Enhancement #3377: Add release binaries for Apple Silicon
   We've added release binaries for macOS on Apple Silicon (M1).
   https://github.com/restic/restic/issues/3377
   https://github.com/restic/restic/pull/3394
 * Enhancement #3414: Add `--keep-within-hourly` option to restic forget
   The `forget` command allowed keeping a given number of hourly backups or to keep all backups
   within a given interval, but it was not possible to specify keeping hourly backups within a
   given interval.
   The new `--keep-within-hourly` option now offers this functionality. Similar options for
   daily/weekly/monthly/yearly are also implemented, the new options are:
   --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly
   <1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h>
   https://github.com/restic/restic/issues/3414
   https://github.com/restic/restic/pull/3416
   https://forum.restic.net/t/forget-policy/4014/11
 * Enhancement #3456: Support filtering and specifying untagged snapshots
   It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag`
   options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches
   snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only
   operate on untagged snapshots.
   https://github.com/restic/restic/issues/3456
   https://github.com/restic/restic/pull/3457
 * Enhancement #3167: Allow specifying limit of `snapshots` list
   The `--last` option allowed limiting the output of the `snapshots` command to the latest
   snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n`
   snapshots.
   This change deprecates the option `--last` in favour of `--latest 1`.
   https://github.com/restic/restic/pull/3167
 * Enhancement #3426: Optimize read performance of mount command
   Reading large files in a mounted repository may be up to five times faster. This improvement
   primarily applies to repositories stored at a backend that can be accessed with low latency,
   like e.g. the local backend.
   https://github.com/restic/restic/pull/3426
 * Enhancement #3427: `find --pack` fallback to index if data file is missing
   When investigating a repository with missing data files, it might be useful to determine
   affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id`
   returned no data as it required accessing the data file. Now, if the necessary data is still
   available in the repository index, it gets retrieved from there.
   The command now also supports looking up multiple pack files in a single `find` run.
   https://github.com/restic/restic/pull/3427
   https://forum.restic.net/t/missing-packs-not-found/2600
 Changelog for restic 0.12.0 (2021-02-14)
 =======================================
@@ -3488,7 +2837,7 @@ Summary
 * Enh #1055: Create subdirs below `data/` for local/sftp backends
 * Enh #1067: Allow loading credentials for s3 from IAM
 * Enh #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
- * Enh #1081: Clarify semantic for `--tag` for the `forget` command
+ * Enh #1081: Clarify semantic for `--tasg` for the `forget` command
 * Enh #1080: Ignore chmod() errors on filesystems which do not support it
 * Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t)
@@ -3532,7 +2881,7 @@ Details
   https://github.com/restic/restic/issues/1073
   https://github.com/restic/restic/pull/1075
- * Enhancement #1081: Clarify semantic for `--tag` for the `forget` command
+ * Enhancement #1081: Clarify semantic for `--tasg` for the `forget` command
   https://github.com/restic/restic/issues/1081
   https://github.com/restic/restic/pull/1090
@@ -13,10 +13,11 @@ bug fixes are most welcome. However even "minor" details as fixing spelling
 errors, improving documentation or pointing out usability issues are a great
 help also.
 The restic project uses the GitHub infrastructure (see the
 [project page](https://github.com/restic/restic)) for all related discussions
 as well as the [forum](https://forum.restic.net/) and the `#restic` channel
-on [irc.libera.chat](https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:6697/#restic).
+on [irc.freenode.net](https://kiwiirc.com/nextclient/irc.freenode.net/restic).
 If you want to find an area that currently needs improving have a look at the
 open issues listed at the
@@ -66,7 +67,7 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
-Restic requires Go version 1.14 or later for compiling. Clone the repo (without
+Restic requires Go version 1.13 or later for compiling. Clone the repo (without
 having `$GOPATH` set) and `cd` into the directory:
    $ unset GOPATH
@@ -123,10 +124,7 @@ down to the following steps:
    writing, ask yourself: If I were the user, what would I need to be aware
    of with this change?
- 8. Do not edit the man pages under `doc/man` or `doc/manual_rest.rst` -
+ 8. Once your code looks good and passes all the tests, we'll merge it. Thanks
    these are autogenerated before new releases.
 9. Once your code looks good and passes all the tests, we'll merge it. Thanks
    a lot for your contribution!
 Please provide the patches for each bug or feature in a separate branch and
@@ -46,8 +46,8 @@ Therefore, restic supports the following backends for storing backups natively:
 - [Local directory](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local)
 - [sftp server (via SSH)](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp)
- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](https://restic.readthedocs.io/en/latest/100_references.html#rest-backend), [rest-server](https://github.com/restic/rest-server))
+- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](doc/100_references.rst#rest-backend), [rest-server](https://github.com/restic/rest-server))
- [Amazon S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
+- [AWS S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
 - [OpenStack Swift](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift)
 - [BackBlaze B2](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2)
 - [Microsoft Azure Blob Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)
@@ -1 +1 @@
-0.13.0
+0.12.0
@@ -35,7 +35,6 @@
 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 //go:build ignore_build_go
 // +build ignore_build_go
 package main
@@ -59,7 +58,7 @@ var config = Config{
 	Main:             "./cmd/restic",                           // package name for the main package
 	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
 	Tests:            []string{"./..."},                        // tests to run
-	MinVersion:       GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
+	MinVersion:       GoVersion{Major: 1, Minor: 11, Patch: 0}, // minimum Go version supported
 }
 // Config configures the build.
@@ -124,8 +123,17 @@ func printEnv(env []string) {
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd string, env map[string]string, args ...string) error {
-	// -trimpath removes all absolute paths from the binary.
+	a := []string{"build"}
-	a := []string{"build", "-trimpath"}
+
 	// try to remove all absolute paths from resulting binary
 	if goVersion.AtLeast(GoVersion{1, 13, 0}) {
 		// use the new flag introduced by Go 1.13
 		a = append(a, "-trimpath")
 	} else {
 		// otherwise try to trim as many paths as possible
 		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
 		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
 	}
 	if enablePIE {
 		a = append(a, "-buildmode=pie")
@@ -1,13 +0,0 @@
 Bugfix: Improve error handling for rclone and REST backend over HTTP2
 When retrieving data from the rclone / REST backend while also using HTTP2
 restic did not detect when no data was returned at all. This could cause
 for example the `check` command to report the following error:
    Pack ID does not match, want [...], got e3b0c442
 This has been fixed by correctly detecting and retrying the incomplete download.
 https://github.com/restic/restic/issues/2742
 https://github.com/restic/restic/pull/3453
 https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
@@ -1,6 +0,0 @@
 Enhancement: Add release binaries for s390x architecture on Linux
 We've added release binaries for Linux using the s390x architecture.
 https://github.com/restic/restic/issues/2780
 https://github.com/restic/restic/pull/3452
@@ -1,11 +0,0 @@
 Bugfix: Fix terminal output redirection for PowerShell
 When redirecting the output of restic using PowerShell on Windows, the
 output contained terminal escape characters. This has been fixed by
 properly detecting the terminal type.
 In addition, the mintty terminal now shows progress output for the backup
 command.
 https://github.com/restic/restic/issues/3111
 https://github.com/restic/restic/pull/3325
@@ -1,9 +0,0 @@
 Bugfix: Treat an empty password as a fatal error for repository init
 When attempting to initialize a new repository, if an empty password was
 supplied, the repository would be created but the init command would return
 an error with a stack trace. Now, if an empty password is provided, it is
 treated as a fatal error, and no repository is created.
 https://github.com/restic/restic/issues/3214
 https://github.com/restic/restic/pull/3283
@@ -1,8 +0,0 @@
 Change: Empty files now have size of 0 in `ls --json` output
 The `ls --json` command used to omit the sizes of empty files in its
 output. It now reports a size of zero explicitly for regular files,
 while omitting the size field for all other types.
 https://github.com/restic/restic/issues/3247
 https://github.com/restic/restic/pull/3257
@@ -1,11 +0,0 @@
 Bugfix: `copy` failed to copy snapshots in rare cases
 The `copy` command could in rare cases fail with the error message `SaveTree(...)
 returned unexpected id ...`.  This has been fixed.
 On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8
 target paths. Note that, due to limitations in the repository format, these are
 not stored properly and should be avoided if possible.
 https://github.com/restic/restic/issues/3267
 https://github.com/restic/restic/pull/3310
@@ -1,11 +0,0 @@
 Bugfix: `backup --quiet` no longer prints status information
 A regression in the latest restic version caused the output of `backup --quiet`
 to contain large amounts of backup progress information when run using an
 interactive terminal. This is fixed now.
 A workaround for this bug is to run restic as follows:
 `restic backup --quiet [..] | cat -`.
 https://github.com/restic/restic/issues/3184
 https://github.com/restic/restic/pull/3186
@@ -1,14 +0,0 @@
 Enhancement: Add `--repository-file2` option to `init` and `copy` command
 The `init` and `copy` command can now be used with the `--repository-file2`
 option or the `$RESTIC_REPOSITORY_FILE2` environment variable.
 These to options are in addition to the `--repo2` flag and allow you to read
 the destination repository from a file.
 Using both `--repository-file` and `--repo2` options resulted in an error for
 the `copy` or `init` command. The handling of this combination of options has
 been fixed. A workaround for this issue is to only use `--repo` or `-r` and
 `--repo2` for `init` or `copy`.
 https://github.com/restic/restic/issues/3293
 https://github.com/restic/restic/pull/3294
@@ -1,7 +0,0 @@
 Bugfix: Fix crash of `check --read-data-subset=x%` run for an empty repository
 The command `restic check --read-data-subset=x%` crashed when run for an empty
 repository. This has been fixed.
 https://github.com/restic/restic/issues/3296
 https://github.com/restic/restic/pull/3309
@@ -1,8 +0,0 @@
 Bugfix: Fix `fdopendir: not a directory` error for local backend
 The `check`, `list packs`, `prune` and `rebuild-index` commands failed
 for the local backend when the `data` folder in the repository contained
 files. This has been fixed.
 https://github.com/restic/restic/issues/3302
 https://github.com/restic/restic/pull/3308
@@ -1,5 +0,0 @@
 Enhancement: Add auto-completion support for fish
 The `generate` command now supports fish auto completion.
 https://github.com/restic/restic/pull/3312
@@ -1,8 +0,0 @@
 Bugfix: Print `created new cache` message only on a terminal
 The message `created new cache` was printed even when the output wasn't a
 terminal. That broke piping `restic dump` output to tar or zip if cache
 directory didn't exist. The message is now only printed on a terminal.
 https://github.com/restic/restic/issues/3334
 https://github.com/restic/restic/pull/3343
@@ -1,8 +0,0 @@
 Enhancement: SFTP backend now checks for disk space
 Backing up over SFTP previously spewed multiple generic "failure" messages
 when the remote disk was full. It now checks for disk space before writing
 a file and fails immediately with a "no space left on device" message.
 https://github.com/restic/restic/issues/3336
 https://github.com/restic/restic/pull/3345
@@ -1,6 +0,0 @@
 Enhancement: Add release binaries for Apple Silicon
 We've added release binaries for macOS on Apple Silicon (M1).
 https://github.com/restic/restic/issues/3377
 https://github.com/restic/restic/pull/3394
@@ -1,7 +0,0 @@
 Bugfix: Fix crash of `backup --exclude='**'`
 The exclude filter `**`, which excludes all files, caused restic to crash. This
 has been corrected.
 https://github.com/restic/restic/issues/3380
 https://github.com/restic/restic/pull/3393
@@ -1,20 +0,0 @@
 Enhancement: Add `--keep-within-hourly` option to restic forget
 The `forget` command allowed keeping a given number of hourly
 backups or to keep all backups within a given interval, but it
 was not possible to specify keeping hourly backups within a given
 interval.
 The new `--keep-within-hourly` option now offers this functionality.
 Similar options for daily/weekly/monthly/yearly are also implemented,
 the new options are:
    --keep-within-hourly <1y2m3d4h>
    --keep-within-daily <1y2m3d4h>
    --keep-within-weekly <1y2m3d4h>
    --keep-within-monthly <1y2m3d4h>
    --keep-within-yearly <1y2m3d4h>
 https://github.com/restic/restic/issues/3414
 https://github.com/restic/restic/pull/3416
 https://forum.restic.net/t/forget-policy/4014/11
@@ -1,9 +0,0 @@
 Enhancement: Support filtering and specifying untagged snapshots
 It was previously not possible to specify an empty tag with the `--tag` and
 `--keep-tag` options. This has now been fixed, such that `--tag ''` and
 `--keep-tag ''` now matches snapshots without tags. This allows e.g. the
 `snapshots` and `forget` commands to only operate on untagged snapshots.
 https://github.com/restic/restic/issues/3456
 https://github.com/restic/restic/pull/3457
@@ -1,9 +0,0 @@
 Enhancement: Allow specifying limit of `snapshots` list
 The `--last` option allowed limiting the output of the `snapshots`
 command to the latest snapshot for each host. The new `--latest n`
 option allows limiting the output to the latest `n` snapshots.
 This change deprecates the option `--last` in favour of `--latest 1`.
 https://github.com/restic/restic/pull/3167
@@ -1,6 +0,0 @@
 Bugfix: Fix possibly missing backup summary of JSON output in case of error
 When using `--json` output it happened from time to time that the summary
 output was missing in case an error occurred. This has been fixed.
 https://github.com/restic/restic/pull/3305
@@ -1,7 +0,0 @@
 Enhancement: Optimize read performance of mount command
 Reading large files in a mounted repository may be up to five times faster.
 This improvement primarily applies to repositories stored at a backend that can
 be accessed with low latency, like e.g. the local backend.
 https://github.com/restic/restic/pull/3426
@@ -1,13 +0,0 @@
 Enhancement: `find --pack` fallback to index if data file is missing
 When investigating a repository with missing data files, it might be useful to
 determine affected snapshots before running `rebuild-index`. Previously,
 `find --pack pack-id` returned no data as it required accessing the data file.
 Now, if the necessary data is still available in the repository index, it gets
 retrieved from there.
 The command now also supports looking up multiple pack files in a single `find`
 run.
 https://github.com/restic/restic/pull/3427
 https://forum.restic.net/t/missing-packs-not-found/2600
@@ -1,8 +0,0 @@
 Bugfix: Correctly handle download errors during `restore`
 Due to a regression in restic 0.12.0, the `restore` command in some cases did
 not retry download errors and only printed a warning. This has been fixed by
 retrying incomplete data downloads.
 https://github.com/restic/restic/issues/3439
 https://github.com/restic/restic/pull/3449
@@ -1,10 +0,0 @@
 Bugfix: Never lock repository for `list locks`
 The `list locks` command previously locked to the repository by default. This
 had the problem that it wouldn't work for an exclusively locked repository and
 that the command would also display its own lock file which can be confusing.
 Now, the `list locks` command never locks the repository.
 https://github.com/restic/restic/issues/1106
 https://github.com/restic/restic/pull/3665
@@ -1,16 +0,0 @@
 Enhancement: Add `--dry-run`/`-n` option to `backup` command
 Testing exclude filters and other configuration options was error prone as
 wrong filters could cause files to be uploaded unintentionally. It was also
 not possible to estimate beforehand how much data would be uploaded.
 The `backup` command now has a `--dry-run`/`-n` option, which performs all the
 normal steps of a backup without actually writing anything to the repository.
 Passing -vv will log information about files that would be added, allowing for
 verification of source and exclusion options before running the real backup.
 https://github.com/restic/restic/issues/1542
 https://github.com/restic/restic/pull/2308
 https://github.com/restic/restic/pull/3210
 https://github.com/restic/restic/pull/3300
@@ -1,15 +0,0 @@
 Enhancement: Add upload checksum for Azure, GS, S3 and Swift backends
 Previously only the B2 and partially the Swift backends verified the integrity
 of uploaded (encrypted) files. The verification works by informing the backend
 about the expected hash of the uploaded file. The backend then verifies the
 upload and thereby rules out any data corruption during upload.
 We have now added upload checksums for the Azure, GS, S3 and Swift backends,
 which besides integrity checking for uploads also means that restic can now be
 used to store backups in S3 buckets which have Object Lock enabled.
 https://github.com/restic/restic/issues/2202
 https://github.com/restic/restic/issues/2700
 https://github.com/restic/restic/issues/3023
 https://github.com/restic/restic/pull/3246
@@ -1,29 +0,0 @@
 Enhancement: Support negative include/exclude patterns
 If a pattern starts with an exclamation mark and it matches a file that was
 previously matched by a regular pattern, the match is cancelled. Notably,
 this can be used with `--exclude-file` to cancel the exclusion of some files.
 It works similarly to `.gitignore`, with the same limitation; Once a directory
 is excluded, it is not possible to include files inside the directory.
 Example of use as an exclude pattern for the `backup` command:
    $HOME/**/*
    !$HOME/Documents
    !$HOME/code
    !$HOME/.emacs.d
    !$HOME/games
    # [...]
    node_modules
    *~
    *.o
    *.lo
    *.pyc
    # [...]
    $HOME/code/linux/*
    !$HOME/code/linux/.git
    # [...]
 https://github.com/restic/restic/issues/233
 https://github.com/restic/restic/pull/2311
@@ -1,13 +0,0 @@
 Bugfix: Make cache crash-resistant and usable by multiple concurrent processes
 The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state
 in the event of restic (or the OS) crashing. This is now less likely to occur
 as files are downloaded to a temporary location before being moved to their
 proper location.
 This also allows multiple concurrent restic processes to operate on a single
 repository without conflicts. Previously, concurrent operations could cause
 segfaults because the processes saw each other's partially downloaded files.
 https://github.com/restic/restic/issues/2345
 https://github.com/restic/restic/pull/2838
@@ -1,10 +0,0 @@
 Enhancement: Add warning for S3 if partial credentials are provided
 Previously restic did not notify about incomplete credentials when using the
 S3 backend, instead just reporting access denied.
 Restic now checks that both the AWS key ID and secret environment variables are
 set before connecting to the remote server, and reports an error if not.
 https://github.com/restic/restic/issues/2388
 https://github.com/restic/restic/pull/3532
@@ -1,11 +0,0 @@
 Bugfix: Improve error handling of repository locking
 Previously, when the lock refresh failed to delete the old lock file, it forgot
 about the newly created one. Instead it continued trying to delete the old
 (usually no longer existing) lock file and thus over time lots of lock files
 accumulated. This has now been fixed.
 https://github.com/restic/restic/issues/2452
 https://github.com/restic/restic/issues/2473
 https://github.com/restic/restic/issues/2562
 https://github.com/restic/restic/pull/3512
@@ -1,8 +0,0 @@
 Enhancement: Support JSON output and quiet mode for the `diff` command
 The `diff` command now supports outputting machine-readable output in JSON
 format. To enable this, pass the `--json` option to the command. To only print
 the summary and suppress detailed output, pass the `--quiet` option.
 https://github.com/restic/restic/issues/2508
 https://github.com/restic/restic/pull/3592
@@ -1,8 +0,0 @@
 Enhancement: Add flag to disable TLS verification for self-signed certificates
 There is now an `--insecure-tls` global option in restic, which disables TLS
 verification for self-signed certificates in order to support some development
 workflows.
 https://github.com/restic/restic/issues/2656
 https://github.com/restic/restic/pull/2657
@@ -1,8 +0,0 @@
 Bugfix: Don't print progress for `backup --json --quiet`
 Unlike the text output, the `--json` output format still printed progress
 information even in `--quiet` mode. This has now been fixed by always
 disabling the progress output in quiet mode.
 https://github.com/restic/restic/issues/2738
 https://github.com/restic/restic/pull/3264
@@ -1,9 +0,0 @@
 Enhancement: Atomic uploads for the SFTP backend
 The SFTP backend did not upload files atomically. An interrupted upload could
 leave an incomplete file behind which could prevent restic from accessing the
 repository. This has now been fixed and uploads in the SFTP backend are done
 atomically.
 https://github.com/restic/restic/issues/3003
 https://github.com/restic/restic/pull/3524
@@ -1,6 +0,0 @@
 Enhancement: Add xattr (extended attributes) support for Solaris
 Restic now supports xattr for the Solaris operating system.
 https://github.com/restic/restic/issues/3127
 https://github.com/restic/restic/pull/3628
@@ -1,8 +0,0 @@
 Bugfix: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
 Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment
 variable, which caused problems in certain system/usage configurations. This
 has now been fixed.
 https://github.com/restic/restic/issues/3382
 https://github.com/restic/restic/pull/3474
@@ -1,10 +0,0 @@
 Enhancement: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
 Restic used to silently ignore the `--no-lock` option of the `forget` command.
 It now skips creation of lock file in case both `--dry-run` and `--no-lock`
 are specified. If `--no-lock` option is specified without `--dry-run`, restic
 prints a warning message to stderr.
 https://github.com/restic/restic/issues/3464
 https://github.com/restic/restic/pull/3623
@@ -1,8 +0,0 @@
 Enhancement: Support random subset by size in `check --read-data-subset`
 The `--read-data-subset` option of the `check` command now supports a third way
 of specifying the subset to check, namely `nS` where `n` is a size in bytes with
 suffix `S` as k/K, m/M, g/G or t/T.
 https://github.com/restic/restic/issues/3490
 https://github.com/restic/restic/pull/3548
@@ -1,8 +0,0 @@
 Bugfix: Make `copy` command honor `--no-lock` for source repository
 The `copy` command previously did not respect the `--no-lock` option for the
 source repository, causing failures with read-only storage backends. This has
 now been fixed such that the option is now respected.
 https://github.com/restic/restic/issues/3518
 https://github.com/restic/restic/pull/3589
@@ -1,9 +0,0 @@
 Enhancement: Improve handling of temporary B2 delete errors
 Deleting files on B2 could sometimes fail temporarily, which required restic to
 retry the delete operation. In some cases the file was deleted nevertheless,
 causing the retries and ultimately the restic command to fail. This has now been
 fixed.
 https://github.com/restic/restic/issues/3541
 https://github.com/restic/restic/pull/3544
@@ -1,8 +0,0 @@
 Enhancement: Add file mode in symbolic notation to `ls --json`
 The `ls --json` command now provides the file mode in symbolic notation (using
 the `permissions` key), aligned with `find --json`.
 https://github.com/restic/restic/issues/3542
 https://github.com/restic/restic/pull/3573
 https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
@@ -1,10 +0,0 @@
 Bugfix: Fix hang with Backblaze B2 on SSL certificate authority error
 Previously, if a request failed with an SSL unknown certificate authority
 error, the B2 backend retried indefinitely and restic would appear to hang.
 This has now been fixed and restic instead fails with an error message.
 https://github.com/restic/restic/issues/3556
 https://github.com/restic/restic/issues/2355
 https://github.com/restic/restic/pull/3571
@@ -1,15 +0,0 @@
 Bugfix: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
 Previously, pressing Ctrl+C in a Windows console where restic was running with
 rclone as the backend would cause rclone to exit prematurely due to getting a
 `SIGINT` signal at the same time as restic. Restic would then wait for a long
 time for time with "unexpected EOF" and "rclone stdio connection already closed"
 errors.
 This has now been fixed by restic starting the rclone process detached from the
 console restic runs in (similar to starting processes in a new process group on
 Linux), which enables restic to gracefully clean up rclone (which now never gets
 the `SIGINT`).
 https://github.com/restic/restic/issues/3601
 https://github.com/restic/restic/pull/3602
@@ -1,14 +0,0 @@
 Change: Ignore parent snapshot for `backup --stdin`
 Restic uses a parent snapshot to speed up directory scanning when performing
 backups, but this only wasted time and memory when the backup source is stdin
 (using the `--stdin` option of the `backup` command), since no directory scanning
 is performed in this case.
 Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows
 restic to skip some startup operations and saves a bit of resources.
 The `--parent` option is still available for `backup --stdin`, but is now ignored.
 https://github.com/restic/restic/issues/3641
 https://github.com/restic/restic/pull/3645
@@ -1,8 +0,0 @@
 Bugfix: The `mount` command now reports symlinks sizes
 Symlinks used to have size zero in restic mountpoints, confusing some
 third-party tools. They now have a size equal to the byte length of their
 target path, as required by POSIX.
 https://github.com/restic/restic/issues/3667
 https://github.com/restic/restic/pull/3668
@@ -1,7 +0,0 @@
 Enhancement: Speed up the `restore --verify` command
 The `--verify` option lets the `restore` command verify the file content
 after it has restored a snapshot. The performance of this operation has
 now been improved by up to a factor of two.
 https://github.com/restic/restic/pull/2594
@@ -1,10 +0,0 @@
 Enhancement: The `backup` command no longer updates file access times on Linux
 When reading files during backup, restic used to cause the operating system to
 update the files' access times. Note that this did not apply to filesystems with
 disabled file access times.
 Restic now instructs the operating system not to update the file access time,
 if the user running restic is the file owner or has root permissions.
 https://github.com/restic/restic/pull/2816
@@ -1,9 +0,0 @@
 Enhancement: Make `recover` collect only unreferenced trees
 Previously, the `recover` command used to generate a snapshot containing *all*
 root trees, even those which were already referenced by a snapshot.
 This has been improved such that it now only processes trees not already
 referenced by any snapshot.
 https://github.com/restic/restic/pull/2880
@@ -1,12 +0,0 @@
 Enhancement: Verify that new or modified keys are stored correctly
 When adding a new key or changing the password of a key, restic used to just
 create the new key (and remove the old one, when changing the password). There
 was no verification that the new key was stored correctly and works properly.
 As the repository cannot be decrypted without a valid key file, this could in
 rare cases cause the repository to become inaccessible.
 Restic now checks that new key files actually work before continuing. This
 can protect against some (rare) cases of hardware or storage problems.
 https://github.com/restic/restic/pull/3429
@@ -1,12 +0,0 @@
 Enhancement: Improve local backend's resilience to (system) crashes
 Restic now ensures that files stored using the `local` backend are created
 atomically (that is, files are either stored completely or not at all). This
 ensures that no incomplete files are left behind even if restic is terminated
 while writing a file.
 In addition, restic now tries to ensure that the directory in the repository
 which contains a newly uploaded file is also written to disk. This can prevent
 missing files if the system crashes or the disk is not properly unmounted.
 https://github.com/restic/restic/pull/3436
@@ -1,9 +0,0 @@
 Bugfix: `rebuild-index` failed if an index file was damaged
 Previously, the `rebuild-index` command would fail with an error if an index
 file was damaged or truncated. This has now been fixed.
 On older restic versions, a (slow) workaround is to use
 `rebuild-index --read-all-packs` or to manually delete the damaged index.
 https://github.com/restic/restic/pull/3488
@@ -1,10 +0,0 @@
 Enhancement: Cache blobs read by the `dump` command
 When dumping a file using the `dump` command, restic did not cache blobs in any
 way, so even consecutive runs of the same blob were loaded from the repository
 again and again, slowing down the dump.
 Now, the caching mechanism already used by the `fuse` command is also used by
 the `dump` command. This makes dumping much faster, especially for sparse files.
 https://github.com/restic/restic/pull/3508
@@ -1,8 +0,0 @@
 Enhancement: Support configurable timeout for the rclone backend
 A slow rclone backend could cause restic to time out while waiting for the
 repository to open. Restic now offers an `-o rclone.timeout` option to make
 this timeout configurable.
 https://github.com/restic/restic/issues/3511
 https://github.com/restic/restic/pull/3514
@@ -1,6 +0,0 @@
 Change: Require Go 1.14 or newer
 Restic now requires Go 1.14 to build. This allows it to use new
 standard library features instead of an external dependency.
 https://github.com/restic/restic/issues/3519
@@ -1,8 +0,0 @@
 Bugfix: Fix handling of `prune --max-repack-size=0`
 Restic ignored the `--max-repack-size` option when passing a value of 0. This
 has now been fixed.
 As a workaround, `--max-repack-size=1` can be used with older versions of restic.
 https://github.com/restic/restic/pull/3591
@@ -1,10 +0,0 @@
 Enhancement: Improve `copy` performance by parallelizing IO
 Restic copy previously only used a single thread for copying blobs between
 repositories, which resulted in limited performance when copying small blobs
 to/from a high latency backend (i.e. any remote backend, especially b2).
 Copying will now use 8 parallel threads to increase the throughput of the copy
 operation.
 https://github.com/restic/restic/pull/3593
@@ -1,11 +0,0 @@
 Bugfix: Avoid choosing parent snapshots newer than time of new snapshot
 The `backup` command, when a `--parent` was not provided, previously chose the
 most recent matching snapshot as the parent snapshot. However, this didn't make
 sense when the user passed `--time` to create a new snapshot older than the most
 recent snapshot.
 Instead, `backup` now chooses the most recent snapshot which is not newer than
 the snapshot-being-created's timestamp, to avoid any time travel.
 https://github.com/restic/restic/pull/3619
@@ -1,4 +1,4 @@
-Enhancement: Clarify semantic for `--tag` for the `forget` command
+Enhancement: Clarify semantic for `--tasg` for the `forget` command
 https://github.com/restic/restic/issues/1081
 https://github.com/restic/restic/pull/1090
@@ -58,7 +58,7 @@ func RunCleanupHandlers() {
 func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
-		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
+		Warnf("%ssignal %v received, cleaning up\n", ClearLine(), s)
 		code := 0
@@ -24,7 +24,8 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui/backup"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/json"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
@@ -59,11 +60,11 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
 		err := runBackup(backupOptions, globalOptions, term, args)
-		t.Kill(nil)
+		if err != nil {
 		if werr := t.Wait(); werr != nil {
 			panic(fmt.Sprintf("term.Run() returned err: %v", err))
 		}
 			return err
 		}
 		t.Kill(nil)
 		return t.Wait()
 	},
 }
@@ -91,25 +92,24 @@ type BackupOptions struct {
 	IgnoreInode             bool
 	IgnoreCtime             bool
 	UseFsSnapshot           bool
 	DryRun                  bool
 }
 var backupOptions BackupOptions
 // ErrInvalidSourceData is used to report an incomplete backup
-var ErrInvalidSourceData = errors.New("at least one source file could not be read")
+var ErrInvalidSourceData = errors.New("failed to read all source data during backup")
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories, and is not newer than the snapshot time)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 	f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
+	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
 	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
 	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
 	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
@@ -132,7 +132,6 @@ func init() {
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
 	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
@@ -472,7 +471,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
 		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
@@ -485,7 +484,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host})
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -526,17 +525,33 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
-	var progressPrinter backup.ProgressPrinter
+	type ArchiveProgressReporter interface {
-	if gopts.JSON {
+		CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration)
-		progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
+		StartFile(filename string)
-	} else {
+		CompleteBlob(filename string, bytes uint64)
-		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
+		ScannerError(item string, fi os.FileInfo, err error) error
-	}
+		ReportTotal(item string, s archiver.ScanStats)
-	progressReporter := backup.NewProgress(progressPrinter)
+		SetMinUpdatePause(d time.Duration)
 		Run(ctx context.Context) error
 		Error(item string, fi os.FileInfo, err error) error
 		Finish(snapshotID restic.ID)
-	if opts.DryRun {
+		// ui.StdioWrapper
-		repo.SetDryRun()
+		Stdout() io.WriteCloser
-		progressReporter.SetDryRun()
+		Stderr() io.WriteCloser
 		// ui.Message
 		E(msg string, args ...interface{})
 		P(msg string, args ...interface{})
 		V(msg string, args ...interface{})
 		VV(msg string, args ...interface{})
 	}
 	var p ArchiveProgressReporter
 	if gopts.JSON {
 		p = json.NewBackup(term, gopts.verbosity)
 	} else {
 		p = ui.NewBackup(term, gopts.verbosity)
 	}
 	// use the terminal for stdout/stderr
@@ -544,14 +559,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	defer func() {
 		gopts.stdout, gopts.stderr = prevStdout, prevStderr
 	}()
-	gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
+	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
-	progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
+	p.SetMinUpdatePause(calculateProgressInterval())
-	t.Go(func() error { return progressReporter.Run(t.Context(gopts.ctx)) })
+	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
 	if !gopts.JSON {
-		progressPrinter.V("lock repository")
+		p.V("lock repository")
 	}
 	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
@@ -572,26 +587,23 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 	if !gopts.JSON {
-		progressPrinter.V("load index files")
+		p.V("load index files")
 	}
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
-	var parentSnapshotID *restic.ID
+	parentSnapshotID, err := findParentSnapshot(gopts.ctx, repo, opts, targets)
 	if !opts.Stdin {
 		parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
 	if err != nil {
 		return err
 	}
 	if !gopts.JSON {
 		if parentSnapshotID != nil {
-				progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
+			p.P("using parent snapshot %v\n", parentSnapshotID.Str())
 		} else {
-				progressPrinter.P("no parent snapshot found, will read all files\n")
+			p.P("no parent snapshot found, will read all files\n")
 			}
 		}
 	}
@@ -620,12 +632,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		}
 		errorHandler := func(item string, err error) error {
-			return progressReporter.Error(item, nil, err)
+			return p.Error(item, nil, err)
 		}
 		messageHandler := func(msg string, args ...interface{}) {
 			if !gopts.JSON {
-				progressPrinter.P(msg, args...)
+				p.P(msg, args...)
 			}
 		}
@@ -635,7 +647,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 	if opts.Stdin {
 		if !gopts.JSON {
-			progressPrinter.V("read data from stdin")
+			p.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
 		targetFS = &fs.Reader{
@@ -650,11 +662,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	sc := archiver.NewScanner(targetFS)
 	sc.SelectByName = selectByNameFilter
 	sc.Select = selectFilter
-	sc.Error = progressReporter.ScannerError
+	sc.Error = p.ScannerError
-	sc.Result = progressReporter.ReportTotal
+	sc.Result = p.ReportTotal
 	if !gopts.JSON {
-		progressPrinter.V("start scan on %v", targets)
+		p.V("start scan on %v", targets)
 	}
 	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
@@ -665,11 +677,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	success := true
 	arch.Error = func(item string, fi os.FileInfo, err error) error {
 		success = false
-		return progressReporter.Error(item, fi, err)
+		return p.Error(item, fi, err)
 	}
-	arch.CompleteItem = progressReporter.CompleteItem
+	arch.CompleteItem = p.CompleteItem
-	arch.StartFile = progressReporter.StartFile
+	arch.StartFile = p.StartFile
-	arch.CompleteBlob = progressReporter.CompleteBlob
+	arch.CompleteBlob = p.CompleteBlob
 	if opts.IgnoreInode {
 		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
@@ -693,30 +705,28 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 	if !gopts.JSON {
-		progressPrinter.V("start backup on %v", targets)
+		p.V("start backup on %v", targets)
 	}
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 	// cleanly shutdown all running goroutines
 	t.Kill(nil)
 	// let's see if one returned an error
-	werr := t.Wait()
+	err = t.Wait()
 	// return original error
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 	// Report finished execution
-	progressReporter.Finish(id)
+	p.Finish(id)
-	if !gopts.JSON && !opts.DryRun {
+	if !gopts.JSON {
-		progressPrinter.P("snapshot %s saved\n", id.Str())
+		p.P("snapshot %s saved\n", id.Str())
 	}
 	if !success {
 		return ErrInvalidSourceData
 	}
 	// Return error if any
-	return werr
+	return err
 }
@@ -5,7 +5,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 	"time"
 	"github.com/restic/restic/internal/cache"
@@ -141,13 +140,8 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
 		}
 		name := entry.Name()
 		if !strings.HasPrefix(name, "restic-check-cache-") {
 			name = name[:10]
 		}
 		tab.AddRow(data{
-			name,
+			entry.Name()[:10],
 			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
 			old,
 			size,
@@ -69,6 +69,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		}
 	}
 	// handle all types that don't need an index
 	switch tpe {
 	case "config":
 		buf, err := json.MarshalIndent(repo.Config(), "", "  ")
@@ -141,7 +142,15 @@ func runCat(gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	}
 	// load index, handle all the other types
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
@@ -158,11 +167,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 	case "blob":
 		err = repo.LoadIndex(gopts.ctx)
 		if err != nil {
 			return err
 		}
 		for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
 			bh := restic.BlobHandle{ID: id, Type: t}
 			if !repo.Index().Has(bh) {
@@ -5,12 +5,10 @@ import (
 	"math/rand"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 	"github.com/spf13/cobra"
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
@@ -56,7 +54,7 @@ func init() {
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
 	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
@@ -67,7 +65,7 @@ func checkFlags(opts CheckOptions) error {
 	}
 	if opts.ReadDataSubset != "" {
 		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		argumentError := errors.Fatal("check flag --read-data-subset has invalid value, please see documentation")
+		argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
 		if err == nil {
 			if len(dataSubset) != 2 {
 				return argumentError
@@ -78,7 +76,7 @@ func checkFlags(opts CheckOptions) error {
 			if dataSubset[1] > totalBucketsMax {
 				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
 			}
-		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
+		} else {
 			percentage, err := parsePercentage(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
@@ -86,19 +84,8 @@ func checkFlags(opts CheckOptions) error {
 			if percentage <= 0.0 || percentage > 100.0 {
 				return errors.Fatal(
-					"check flag --read-data-subset=x% x must be above 0.0% and at most 100.0%")
+					"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
 			}
 		} else {
 			fileSize, err := parseSizeStr(opts.ReadDataSubset)
 			if err != nil {
 				return argumentError
 			}
 			if fileSize <= 0.0 {
 				return errors.Fatal(
 					"check flag --read-data-subset=n n must be above 0")
 			}
 		}
 	}
@@ -159,9 +146,6 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 	}
 	cachedir := gopts.CacheDir
 	if cachedir == "" {
 		cachedir = cache.EnvDir()
 	}
 	// use a cache in a temporary directory
 	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
@@ -257,11 +241,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
 		defer bar.Done()
 		chkr.Structure(gopts.ctx, bar, errChan)
@@ -279,11 +259,6 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 	// Wait for the progress bar to be complete before printing more below.
 	// Must happen after `errChan` is read from in the above loop to avoid
 	// deadlocking in the case of errors.
 	wg.Wait()
 	if opts.CheckUnused {
 		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
 			Verbosef("unused blob %v\n", id)
@@ -319,28 +294,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
 			packCount := uint64(len(packs))
 			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
-		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
+		} else {
-			percentage, err := parsePercentage(opts.ReadDataSubset)
+			percentage, _ := parsePercentage(opts.ReadDataSubset)
 			if err == nil {
 			packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
 			Verbosef("read %.1f%% of data packs\n", percentage)
 		}
 		} else {
 			repoSize := int64(0)
 			allPacks := chkr.GetPacks()
 			for _, size := range allPacks {
 				repoSize += size
 			}
 			if repoSize == 0 {
 				return errors.Fatal("Cannot read from a repository having size 0")
 			}
 			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
 			if subsetSize > repoSize {
 				subsetSize = repoSize
 			}
 			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
 			Verbosef("read %d bytes of data packs\n", subsetSize)
 		}
 		if packs == nil {
 			return errors.Fatal("internal error: failed to select packs to check")
 		}
@@ -373,7 +331,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint
 func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
 	packCount := len(allPacks)
 	packsToCheck := int(float64(packCount) * (percentage / 100.0))
-	if packCount > 0 && packsToCheck < 1 {
+	if packsToCheck < 1 {
 		packsToCheck = 1
 	}
 	timeNs := time.Now().UnixNano()
@@ -391,11 +349,6 @@ func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage floa
 		id := keys[idx[i]]
 		packs[id] = allPacks[id]
 	}
 	return packs
 }
 func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, repoSize int64) map[restic.ID]int64 {
 	subsetPercentage := (float64(subsetSize) / float64(repoSize)) * 100.0
 	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }
@@ -122,44 +122,3 @@ func TestSelectRandomPacksByPercentage(t *testing.T) {
 		rtest.Assert(t, ok, "Expected input and output to be equal")
 	}
 }
 func TestSelectNoRandomPacksByPercentage(t *testing.T) {
 	// that the a repository without pack files works
 	var testPacks = make(map[restic.ID]int64)
 	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
 }
 func TestSelectRandomPacksByFileSize(t *testing.T) {
 	var testPacks = make(map[restic.ID]int64)
 	for i := 1; i <= 10; i++ {
 		id := restic.NewRandomID()
 		// ensure unique ids
 		id[0] = byte(i)
 		testPacks[id] = 0
 	}
 	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
 	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
 	selectedPacks = selectRandomPacksByFileSize(testPacks, 10240, 51200)
 	rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
 	for pack := range selectedPacks {
 		_, ok := testPacks[pack]
 		rtest.Assert(t, ok, "Unexpected selection")
 	}
 	selectedPacks = selectRandomPacksByFileSize(testPacks, 500, 500)
 	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
 	for pack := range selectedPacks {
 		_, ok := testPacks[pack]
 		rtest.Assert(t, ok, "Unexpected item in selection")
 	}
 }
 func TestSelectNoRandomPacksByFileSize(t *testing.T) {
 	// that the a repository without pack files works
 	var testPacks = make(map[restic.ID]int64)
 	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
 }
@@ -73,13 +73,11 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 	if !gopts.NoLock {
 	srcLock, err := lockRepo(ctx, srcRepo)
 	defer unlockRepo(srcLock)
 	if err != nil {
 		return err
 	}
 	}
 	dstLock, err := lockRepo(ctx, dstRepo)
 	defer unlockRepo(dstLock)
@@ -176,12 +174,9 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 const numCopyWorkers = 8
 func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
 	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
 	idChan := make(chan restic.ID)
 	wg, ctx := errgroup.WithContext(ctx)
 	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
@@ -191,9 +186,9 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 	}, nil)
 	wg.Go(func() error {
 		defer close(idChan)
 		// reused buffer
 		var buf []byte
 		for tree := range treeStream {
 			if tree.Error != nil {
 				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
@@ -201,39 +196,22 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 			// Do we already have this tree blob?
 			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
-				// copy raw tree bytes to avoid problems if the serialization changes
+				newTreeID, err := dstRepo.SaveTree(ctx, tree.Tree)
 				var err error
 				buf, err = srcRepo.LoadBlob(ctx, restic.TreeBlob, tree.ID, buf)
 				if err != nil {
-					return fmt.Errorf("LoadBlob(%v) for tree returned error %v", tree.ID, err)
+					return fmt.Errorf("SaveTree(%v) returned error %v", tree.ID.Str(), err)
 				}
 				// Assurance only.
 				if newTreeID != tree.ID {
 					return fmt.Errorf("SaveTree(%v) returned unexpected id %s", tree.ID.Str(), newTreeID.Str())
 				}
 			}
-				_, _, err = dstRepo.SaveBlob(ctx, restic.TreeBlob, buf, tree.ID, false)
+			// TODO: parallelize blob down/upload
 				if err != nil {
 					return fmt.Errorf("SaveBlob(%v) for tree returned error %v", tree.ID.Str(), err)
 				}
 			}
 			for _, entry := range tree.Nodes {
 				// Recursion into directories is handled by StreamTrees
 				// Copy the blobs for this file.
 				for _, blobID := range entry.Content {
 					select {
 					case idChan <- blobID:
 					case <-ctx.Done():
 						return ctx.Err()
 					}
 				}
 			}
 		}
 		return nil
 	})
 	for i := 0; i < numCopyWorkers; i++ {
 		wg.Go(func() error {
 			// reused buffer
 			var buf []byte
 			for blobID := range idChan {
 					// Do we already have this data blob?
 					if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
 						continue
@@ -250,8 +228,10 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
 						return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
 					}
 				}
 			}
 		}
 		return nil
 	})
 	}
 	return wg.Wait()
 }
@@ -4,21 +4,12 @@ package main
 import (
 	"context"
 	"crypto/aes"
 	"crypto/cipher"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"runtime"
 	"sort"
 	"time"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
@@ -48,17 +39,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 var tryRepair bool
 var repairByte bool
 var extractPack bool
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
 	cmdDebug.AddCommand(cmdDebugExamine)
 	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
 	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
 	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -182,367 +165,3 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("no such type %q", tpe)
 	}
 }
 var cmdDebugExamine = &cobra.Command{
 	Use:               "examine pack-ID...",
 	Short:             "Examine a pack file",
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runDebugExamine(globalOptions, args)
 	},
 }
 func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte {
 	if bytewise {
 		Printf("        trying to repair blob by finding a broken byte\n")
 	} else {
 		Printf("        trying to repair blob with single bit flip\n")
 	}
 	ch := make(chan int)
 	var wg errgroup.Group
 	done := make(chan struct{})
 	var fixed []byte
 	var found bool
 	workers := runtime.GOMAXPROCS(0)
 	Printf("         spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
 	for i := 0; i < workers; i++ {
 		wg.Go(func() error {
 			// make a local copy of the buffer
 			buf := make([]byte, len(input))
 			copy(buf, input)
 			testFlip := func(idx int, pattern byte) bool {
 				// flip bits
 				buf[idx] ^= pattern
 				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
 				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
 				if err == nil {
 					Printf("\n")
 					Printf("        blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
 					Printf("        hash is %v\n", restic.Hash(plaintext))
 					close(done)
 					found = true
 					fixed = plaintext
 					return true
 				}
 				// flip bits back
 				buf[idx] ^= pattern
 				return false
 			}
 			for i := range ch {
 				if bytewise {
 					for j := 0; j < 255; j++ {
 						if testFlip(i, byte(j)) {
 							return nil
 						}
 					}
 				} else {
 					for j := 0; j < 7; j++ {
 						// flip each bit once
 						if testFlip(i, (1 << uint(j))) {
 							return nil
 						}
 					}
 				}
 			}
 			return nil
 		})
 	}
 	wg.Go(func() error {
 		defer close(ch)
 		start := time.Now()
 		info := time.Now()
 		for i := range input {
 			select {
 			case ch <- i:
 			case <-done:
 				Printf("     done after %v\n", time.Since(start))
 				return nil
 			}
 			if time.Since(info) > time.Second {
 				secs := time.Since(start).Seconds()
 				gps := float64(i) / secs
 				remaining := len(input) - i
 				eta := time.Duration(float64(remaining)/gps) * time.Second
 				Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
 					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
 				info = time.Now()
 			}
 		}
 		return nil
 	})
 	err := wg.Wait()
 	if err != nil {
 		panic("all go rountines can only return nil")
 	}
 	if !found {
 		Printf("\n        blob could not be repaired\n")
 	}
 	return fixed
 }
 func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
 	// strip signature at the end
 	l := len(buf)
 	nonce, ct := buf[:16], buf[16:l-16]
 	out := make([]byte, len(ct))
 	c, err := aes.NewCipher(k.EncryptionKey[:])
 	if err != nil {
 		panic(fmt.Sprintf("unable to create cipher: %v", err))
 	}
 	e := cipher.NewCTR(c, nonce)
 	e.XORKeyStream(out, ct)
 	return out
 }
 func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
 	be := repo.Backend()
 	h := restic.Handle{
 		Name: pack.String(),
 		Type: restic.PackFile,
 	}
 	for _, blob := range list {
 		Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
 		buf := make([]byte, blob.Length)
 		err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
 			n, err := io.ReadFull(rd, buf)
 			if err != nil {
 				return fmt.Errorf("read error after %d bytes: %v", n, err)
 			}
 			return nil
 		})
 		if err != nil {
 			Warnf("error read: %v\n", err)
 			continue
 		}
 		key := repo.Key()
 		nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
 		plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
 		if err != nil {
 			Warnf("error decrypting blob: %v\n", err)
 			var plain []byte
 			if tryRepair || repairByte {
 				plain = tryRepairWithBitflip(ctx, key, buf, repairByte)
 			}
 			var prefix string
 			if plain != nil {
 				id := restic.Hash(plain)
 				if !id.Equal(blob.ID) {
 					Printf("         repaired blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plain), id, blob.ID)
 					prefix = "repaired-wrong-hash-"
 				} else {
 					Printf("         successfully repaired blob (length %v), hash is %v, ID matches\n", len(plain), id)
 					prefix = "repaired-"
 				}
 			} else {
 				plain = decryptUnsigned(ctx, key, buf)
 				prefix = "damaged-"
 			}
 			err = storePlainBlob(blob.ID, prefix, plain)
 			if err != nil {
 				return err
 			}
 			continue
 		}
 		id := restic.Hash(plaintext)
 		var prefix string
 		if !id.Equal(blob.ID) {
 			Printf("         successfully decrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", len(plaintext), id, blob.ID)
 			prefix = "wrong-hash-"
 		} else {
 			Printf("         successfully decrypted blob (length %v), hash is %v, ID matches\n", len(plaintext), id)
 			prefix = "correct-"
 		}
 		if extractPack {
 			err = storePlainBlob(id, prefix, plaintext)
 			if err != nil {
 				return err
 			}
 		}
 	}
 	return nil
 }
 func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 	filename := fmt.Sprintf("%s%s.bin", prefix, id)
 	f, err := os.Create(filename)
 	if err != nil {
 		return err
 	}
 	_, err = f.Write(plain)
 	if err != nil {
 		_ = f.Close()
 		return err
 	}
 	err = f.Close()
 	if err != nil {
 		return err
 	}
 	Printf("decrypt of blob %v stored at %v\n", id, filename)
 	return nil
 }
 func runDebugExamine(gopts GlobalOptions, args []string) error {
 	ids := make([]restic.ID, 0)
 	for _, name := range args {
 		id, err := restic.ParseID(name)
 		if err != nil {
 			Warnf("error: %v\n", err)
 			continue
 		}
 		ids = append(ids, id)
 	}
 	if len(ids) == 0 {
 		return errors.Fatal("no pack files to examine")
 	}
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
 	}
 	if !gopts.NoLock {
 		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
 	}
 	for _, id := range ids {
 		err := examinePack(gopts.ctx, repo, id)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		}
 		if err == context.Canceled {
 			break
 		}
 	}
 	return nil
 }
 func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
 	Printf("examine %v\n", id)
 	h := restic.Handle{
 		Type: restic.PackFile,
 		Name: id.String(),
 	}
 	fi, err := repo.Backend().Stat(ctx, h)
 	if err != nil {
 		return err
 	}
 	Printf("  file size is %v\n", fi.Size)
 	buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
 	if err != nil {
 		return err
 	}
 	gotID := restic.Hash(buf)
 	if !id.Equal(gotID) {
 		Printf("  wanted hash %v, got %v\n", id, gotID)
 	} else {
 		Printf("  hash for file content matches\n")
 	}
 	Printf("  ========================================\n")
 	Printf("  looking for info in the indexes\n")
 	blobsLoaded := false
 	// examine all data the indexes have for the pack file
 	for _, idx := range repo.Index().(*repository.MasterIndex).All() {
 		idxIDs, err := idx.IDs()
 		if err != nil {
 			idxIDs = restic.IDs{}
 		}
 		blobs := idx.ListPack(id)
 		if len(blobs) == 0 {
 			continue
 		}
 		Printf("    index %v:\n", idxIDs)
 		// convert list of blobs to []restic.Blob
 		var list []restic.Blob
 		for _, b := range blobs {
 			list = append(list, b.Blob)
 		}
 		checkPackSize(list, fi.Size)
 		err = loadBlobs(ctx, repo, id, list)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		} else {
 			blobsLoaded = true
 		}
 	}
 	Printf("  ========================================\n")
 	Printf("  inspect the pack itself\n")
 	blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), fi.Size)
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}
 	checkPackSize(blobs, fi.Size)
 	if !blobsLoaded {
 		return loadBlobs(ctx, repo, id, blobs)
 	}
 	return nil
 }
 func checkPackSize(blobs []restic.Blob, fileSize int64) {
 	// track current size and offset
 	var size, offset uint64
 	sort.Slice(blobs, func(i, j int) bool {
 		return blobs[i].Offset < blobs[j].Offset
 	})
 	for _, pb := range blobs {
 		Printf("      %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
 		if offset != uint64(pb.Offset) {
 			Printf("      hole in file, want offset %v, got %v\n", offset, pb.Offset)
 		}
 		offset += uint64(pb.Length)
 		size += uint64(pb.Length)
 	}
 	// compute header size, per blob: 1 byte type, 4 byte length, 32 byte id
 	size += uint64(restic.CiphertextLength(len(blobs) * (1 + 4 + 32)))
 	// length in uint32 little endian
 	size += 4
 	if uint64(fileSize) != size {
 		Printf("      file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
 	} else {
 		Printf("      file sizes match\n")
 	}
 }
@@ -2,7 +2,6 @@ package main
 import (
 	"context"
 	"encoding/json"
 	"path"
 	"reflect"
 	"sort"
@@ -65,27 +64,13 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string)
 type Comparer struct {
 	repo restic.Repository
 	opts DiffOptions
 	printChange func(change *Change)
 }
 type Change struct {
 	MessageType string `json:"message_type"` // "change"
 	Path        string `json:"path"`
 	Modifier    string `json:"modifier"`
 }
 func NewChange(path string, mode string) *Change {
 	return &Change{MessageType: "change", Path: path, Modifier: mode}
 }
 // DiffStat collects stats for all types of items.
 type DiffStat struct {
-	Files     int    `json:"files"`
+	Files, Dirs, Others  int
-	Dirs      int    `json:"dirs"`
+	DataBlobs, TreeBlobs int
-	Others    int    `json:"others"`
+	Bytes                uint64
 	DataBlobs int    `json:"data_blobs"`
 	TreeBlobs int    `json:"tree_blobs"`
 	Bytes     uint64 `json:"bytes"`
 }
 // Add adds stats information for node to s.
@@ -128,14 +113,21 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 	}
 }
-type DiffStatsContainer struct {
+// DiffStats collects the differences between two snapshots.
-	MessageType                          string         `json:"message_type"` // "statistics"
+type DiffStats struct {
-	SourceSnapshot                       string         `json:"source_snapshot"`
+	ChangedFiles                         int
-	TargetSnapshot                       string         `json:"target_snapshot"`
+	Added                                DiffStat
-	ChangedFiles                         int            `json:"changed_files"`
+	Removed                              DiffStat
-	Added                                DiffStat       `json:"added"`
+	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
-	Removed                              DiffStat       `json:"removed"`
+}
-	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
+
 // NewDiffStats creates new stats for a diff run.
 func NewDiffStats() *DiffStats {
 	return &DiffStats{
 		BlobsBefore: restic.NewBlobSet(),
 		BlobsAfter:  restic.NewBlobSet(),
 		BlobsCommon: restic.NewBlobSet(),
 	}
 }
 // updateBlobs updates the blob counters in the stats struct.
@@ -170,7 +162,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		if node.Type == "dir" {
 			name += "/"
 		}
-		c.printChange(NewChange(name, "+"))
+		Printf("%-5s%v\n", mode, name)
 		stats.Add(node)
 		addBlobs(blobs, node)
@@ -229,7 +221,7 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
 	return tree1Nodes, tree2Nodes, uniqueNames
 }
-func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
+func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
 	tree1, err := c.repo.LoadTree(ctx, id1)
 	if err != nil {
@@ -273,7 +265,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			}
 			if mod != "" {
-				c.printChange(NewChange(name, mod))
+				Printf("%-5s%v\n", mod, name)
 			}
 			if node1.Type == "dir" && node2.Type == "dir" {
@@ -292,7 +284,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			if node1.Type == "dir" {
 				prefix += "/"
 			}
-			c.printChange(NewChange(prefix, "-"))
+			Printf("%-5s%v\n", "-", prefix)
 			stats.Removed.Add(node1)
 			if node1.Type == "dir" {
@@ -306,7 +298,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 			if node2.Type == "dir" {
 				prefix += "/"
 			}
-			c.printChange(NewChange(prefix, "+"))
+			Printf("%-5s%v\n", "+", prefix)
 			stats.Added.Add(node2)
 			if node2.Type == "dir" {
@@ -356,9 +348,7 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 	if !gopts.JSON {
 	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
 	}
 	if sn1.Tree == nil {
 		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
@@ -371,33 +361,9 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,
 		printChange: func(change *Change) {
 			Printf("%-5s%v\n", change.Modifier, change.Path)
 		},
 	}
-	if gopts.JSON {
+	stats := NewDiffStats()
 		enc := json.NewEncoder(gopts.stdout)
 		c.printChange = func(change *Change) {
 			err := enc.Encode(change)
 			if err != nil {
 				Warnf("JSON encode failed: %v\n", err)
 			}
 		}
 	}
 	if gopts.Quiet {
 		c.printChange = func(change *Change) {}
 	}
 	stats := &DiffStatsContainer{
 		MessageType:    "statistics",
 		SourceSnapshot: args[0],
 		TargetSnapshot: args[1],
 		BlobsBefore:    restic.NewBlobSet(),
 		BlobsAfter:     restic.NewBlobSet(),
 		BlobsCommon:    restic.NewBlobSet(),
 	}
 	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
 	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
@@ -410,12 +376,6 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
 	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
 	if gopts.JSON {
 		err := json.NewEncoder(gopts.stdout).Encode(stats)
 		if err != nil {
 			Warnf("JSON encode failed: %v\n", err)
 		}
 	} else {
 	Printf("\n")
 	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
 	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
@@ -424,7 +384,6 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
 	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
 	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
 	}
 	return nil
 }
@@ -67,31 +67,41 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
 	if tree == nil {
 		return fmt.Errorf("called with a nil tree")
 	}
 	if repo == nil {
 		return fmt.Errorf("called with a nil repository")
 	}
 	l := len(pathComponents)
 	if l == 0 {
 		return fmt.Errorf("empty path components")
 	}
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
 		if err := checkStdoutArchive(); err != nil {
 			return err
 		}
-		return d.DumpTree(ctx, tree, "/")
+		return writeDump(ctx, repo, tree, "/", os.Stdout)
 	}
 	item := filepath.Join(prefix, pathComponents[0])
 	l := len(pathComponents)
 	for _, node := range tree.Nodes {
 		// If dumping something in the highest level it will just take the
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {
 			switch {
 			case l == 1 && dump.IsFile(node):
-				return d.WriteNode(ctx, node)
+				return dump.GetNodeData(ctx, os.Stdout, repo, node)
 			case l > 1 && dump.IsDir(node):
 				subtree, err := repo.LoadTree(ctx, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
 			case dump.IsDir(node):
 				if err := checkStdoutArchive(); err != nil {
 					return err
@@ -100,7 +110,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return err
 				}
-				return d.DumpTree(ctx, subtree, item)
+				return writeDump(ctx, repo, subtree, item, os.Stdout)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
 			case !dump.IsFile(node):
@@ -118,8 +128,12 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 	var wd dump.WriteDump
 	switch opts.Archive {
-	case "tar", "zip":
+	case "tar":
 		wd = dump.WriteTar
 	case "zip":
 		wd = dump.WriteZip
 	default:
 		return fmt.Errorf("unknown archive format %q", opts.Archive)
 	}
@@ -152,7 +166,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -173,8 +187,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
-	d := dump.New(opts.Archive, repo, os.Stdout)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
 	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
 	if err != nil {
 		Exitf(2, "cannot dump file: %v", err)
 	}
@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
 	"sort"
 	"strings"
 	"time"
@@ -41,6 +40,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 const shortStr = 8 // Length of short IDs: 4 bytes as hex strings
 // FindOptions bundles all options for the find command.
 type FindOptions struct {
 	Oldest             string
@@ -267,7 +268,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
 			return false, walker.ErrSkipNode
 		}
@@ -351,7 +352,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
+			Printf("Unable to load tree %s\n ... which belongs to snapshot %s.\n", parentTreeID, sn.ID())
 			return false, walker.ErrSkipNode
 		}
@@ -385,12 +386,12 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 				idStr := id.String()
 				if _, ok := f.blobIDs[idStr]; !ok {
 					// Look for short ID form
-					if _, ok := f.blobIDs[id.Str()]; !ok {
+					if _, ok := f.blobIDs[idStr[:shortStr]]; !ok {
 						continue
 					}
 					// Replace the short ID with the long one
 					f.blobIDs[idStr] = struct{}{}
-					delete(f.blobIDs, id.Str())
+					delete(f.blobIDs, idStr[:shortStr])
 				}
 				f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
 			}
@@ -400,8 +401,6 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	})
 }
 var errAllPacksFound = errors.New("all packs found")
 // packsToBlobs converts the list of pack IDs to a list of blob IDs that
 // belong to those packs.
 func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
@@ -413,18 +412,20 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 		f.blobIDs = make(map[string]struct{})
 	}
 	allPacksFound := false
 	packsFound := 0
 	debug.Log("Looking for packs...")
 	err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		if allPacksFound {
 			return nil
 		}
 		idStr := id.String()
 		if _, ok := packIDs[idStr]; !ok {
 			// Look for short ID form
-			if _, ok := packIDs[id.Str()]; !ok {
+			if _, ok := packIDs[idStr[:shortStr]]; !ok {
 				return nil
 			}
 			delete(packIDs, id.Str())
 		} else {
 			// forget found id
 			delete(packIDs, idStr)
 		}
 		debug.Log("Found pack %s", idStr)
 		blobs, _, err := f.repo.ListPack(ctx, id, size)
@@ -435,75 +436,25 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 			f.blobIDs[b.ID.String()] = struct{}{}
 		}
 		// Stop searching when all packs have been found
-		if len(packIDs) == 0 {
+		packsFound++
-			return errAllPacksFound
+		if packsFound >= len(packIDs) {
 			allPacksFound = true
 		}
 		return nil
 	})
-	if err != nil && err != errAllPacksFound {
+	if err != nil {
 		return err
 	}
-	if err != errAllPacksFound {
+	if !allPacksFound {
-		// try to resolve unknown pack ids from the index
+		return errors.Fatal("unable to find all specified pack(s)")
 		packIDs = f.indexPacksToBlobs(ctx, packIDs)
 	}
 	if len(packIDs) > 0 {
 		list := make([]string, 0, len(packIDs))
 		for h := range packIDs {
 			list = append(list, h)
 		}
 		sort.Strings(list)
 		return errors.Fatalf("unable to find pack(s): %v", list)
 	}
 	debug.Log("%d blobs found", len(f.blobIDs))
 	return nil
 }
 func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
 	wctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	// remember which packs were found in the index
 	indexPackIDs := make(map[string]struct{})
 	for pb := range f.repo.Index().Each(wctx) {
 		idStr := pb.PackID.String()
 		// keep entry in packIDs as Each() returns individual index entries
 		matchingID := false
 		if _, ok := packIDs[idStr]; ok {
 			matchingID = true
 		} else {
 			if _, ok := packIDs[pb.PackID.Str()]; ok {
 				// expand id
 				delete(packIDs, pb.PackID.Str())
 				packIDs[idStr] = struct{}{}
 				matchingID = true
 			}
 		}
 		if matchingID {
 			f.blobIDs[pb.ID.String()] = struct{}{}
 			indexPackIDs[idStr] = struct{}{}
 		}
 	}
 	for id := range indexPackIDs {
 		delete(packIDs, id)
 	}
 	if len(indexPackIDs) > 0 {
 		list := make([]string, 0, len(indexPackIDs))
 		for h := range indexPackIDs {
 			list = append(list, h)
 		}
 		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
 	}
 	return packIDs
 }
 func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobType) {
 	idx := f.repo.Index()
@@ -612,7 +563,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	}
 	if opts.PackID {
-		err := f.packsToBlobs(ctx, f.pat.pattern)
+		err := f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
 		if err != nil {
 			return err
 		}
@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"io"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -16,9 +15,8 @@ var cmdForget = &cobra.Command{
 	Long: `
 The "forget" command removes snapshots according to a policy. Please note that
 this command really only deletes the snapshot object in the repository, which
-is a reference to data stored there. In order to remove the unreferenced data
+is a reference to data stored there. In order to remove this (now unreferenced)
-after "forget" was run successfully, see the "prune" command. Please also read
+data after 'forget' was run successfully, see the 'prune' command.
 the documentation for "forget" to learn about important security considerations.
 EXIT STATUS
 ===========
@@ -40,11 +38,6 @@ type ForgetOptions struct {
 	Monthly  int
 	Yearly   int
 	Within   restic.Duration
 	WithinHourly  restic.Duration
 	WithinDaily   restic.Duration
 	WithinWeekly  restic.Duration
 	WithinMonthly restic.Duration
 	WithinYearly  restic.Duration
 	KeepTags restic.TagLists
 	Hosts   []string
@@ -71,11 +64,6 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinHourly, "keep-within-hourly", "", "keep hourly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinDaily, "keep-within-daily", "", "keep daily snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
 	f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
@@ -110,17 +98,11 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 	if gopts.NoLock && !opts.DryRun {
 		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
 	}
 	if !opts.DryRun || !gopts.NoLock {
 	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 	}
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
@@ -153,11 +135,6 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 			Monthly: opts.Monthly,
 			Yearly:  opts.Yearly,
 			Within:  opts.Within,
 			WithinHourly:  opts.WithinHourly,
 			WithinDaily:   opts.WithinDaily,
 			WithinWeekly:  opts.WithinWeekly,
 			WithinMonthly: opts.WithinMonthly,
 			WithinYearly:  opts.WithinYearly,
 			Tags:    opts.KeepTags,
 		}
@@ -10,10 +10,10 @@ import (
 var cmdGenerate = &cobra.Command{
 	Use:   "generate [flags]",
-	Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
+	Short: "Generate manual pages and auto-completion files (bash, zsh)",
 	Long: `
 The "generate" command writes automatically generated files (like the man pages
-and the auto-completion files for bash, fish and zsh).
+and the auto-completion files for bash and zsh).
 EXIT STATUS
 ===========
@@ -27,7 +27,6 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type generateOptions struct {
 	ManDir             string
 	BashCompletionFile string
 	FishCompletionFile string
 	ZSHCompletionFile  string
 }
@@ -38,7 +37,6 @@ func init() {
 	fs := cmdGenerate.Flags()
 	fs.StringVar(&genOpts.ManDir, "man", "", "write man pages to `directory`")
 	fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
 	fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
 	fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
 }
@@ -65,11 +63,6 @@ func writeBashCompletion(file string) error {
 	return cmdRoot.GenBashCompletionFile(file)
 }
 func writeFishCompletion(file string) error {
 	Verbosef("writing fish completion file to %v\n", file)
 	return cmdRoot.GenFishCompletionFile(file, true)
 }
 func writeZSHCompletion(file string) error {
 	Verbosef("writing zsh completion file to %v\n", file)
 	return cmdRoot.GenZshCompletionFile(file)
@@ -90,13 +83,6 @@ func runGenerate(cmd *cobra.Command, args []string) error {
 		}
 	}
 	if genOpts.FishCompletionFile != "" {
 		err := writeFishCompletion(genOpts.FishCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 	if genOpts.ZSHCompletionFile != "" {
 		err := writeZSHCompletion(genOpts.ZSHCompletionFile)
 		if err != nil {
@@ -53,6 +53,11 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 	be, err := create(repo, gopts.extended)
 	if err != nil {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 	gopts.password, err = ReadPasswordTwice(gopts,
 		"enter password for new repository: ",
 		"enter password again: ")
@@ -60,11 +65,6 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 	be, err := create(repo, gopts.extended)
 	if err != nil {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 	s := repository.New(be)
 	err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)
@@ -131,11 +131,6 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
 	if err != nil {
 		return err
 	}
 	Verbosef("saved new key as %s\n", id)
 	return nil
@@ -166,14 +161,8 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 	oldID := repo.KeyName()
-	err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
 	if err != nil {
 		return err
 	}
 	h := restic.Handle{Type: restic.KeyFile, Name: oldID}
 	err = repo.Backend().Remove(gopts.ctx, h)
 	if err != nil {
 		return err
@@ -184,19 +173,6 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 	return nil
 }
 func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
 	// Verify new key to make sure it really works. A broken key can render the
 	// whole repository inaccessible
 	err := repo.SearchKey(ctx, pw, 0, key.Name())
 	if err != nil {
 		// the key is invalid, try to remove it
 		h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
 		_ = repo.Backend().Remove(ctx, h)
 		return errors.Fatalf("failed to access repository with new key: %v", err)
 	}
 	return nil
 }
 func runKey(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
 		return errors.Fatal("wrong number of arguments")
@@ -39,7 +39,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 		return err
 	}
-	if !opts.NoLock && args[0] != "locks" {
+	if !opts.NoLock {
 		lock, err := lockRepo(opts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
@@ -61,9 +61,9 @@ func init() {
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
-	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 }
@@ -74,49 +74,23 @@ type lsSnapshot struct {
 	StructType string     `json:"struct_type"` // "snapshot"
 }
-// Print node in our custom JSON format, followed by a newline.
+type lsNode struct {
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	n := &struct {
 	Name       string      `json:"name"`
 	Type       string      `json:"type"`
 	Path       string      `json:"path"`
 	UID        uint32      `json:"uid"`
 	GID        uint32      `json:"gid"`
-		Size        *uint64     `json:"size,omitempty"`
+	Size       uint64      `json:"size,omitempty"`
 	Mode       os.FileMode `json:"mode,omitempty"`
 		Permissions string      `json:"permissions,omitempty"`
 	ModTime    time.Time   `json:"mtime,omitempty"`
 	AccessTime time.Time   `json:"atime,omitempty"`
 	ChangeTime time.Time   `json:"ctime,omitempty"`
 	StructType string      `json:"struct_type"` // "node"
 		size uint64 // Target for Size pointer.
 	}{
 		Name:        node.Name,
 		Type:        node.Type,
 		Path:        path,
 		UID:         node.UID,
 		GID:         node.GID,
 		size:        node.Size,
 		Mode:        node.Mode,
 		Permissions: node.Mode.String(),
 		ModTime:     node.ModTime,
 		AccessTime:  node.AccessTime,
 		ChangeTime:  node.ChangeTime,
 		StructType:  "node",
 	}
 	// Always print size for regular files, even when empty,
 	// but never for other types.
 	if node.Type == "file" {
 		n.Size = &n.size
 	}
 	return enc.Encode(n)
 }
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
+		return errors.Fatal("no snapshot ID specified")
 	}
 	// extract any specific directories to walk
@@ -185,7 +159,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		enc := json.NewEncoder(gopts.stdout)
 		printSnapshot = func(sn *restic.Snapshot) {
-			err := enc.Encode(lsSnapshot{
+			err = enc.Encode(lsSnapshot{
 				Snapshot:   sn,
 				ID:         sn.ID(),
 				ShortID:    sn.ID().Str(),
@@ -197,7 +171,19 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		}
 		printNode = func(path string, node *restic.Node) {
-			err := lsNodeJSON(enc, path, node)
+			err = enc.Encode(lsNode{
 				Name:       node.Name,
 				Type:       node.Type,
 				Path:       path,
 				UID:        node.UID,
 				GID:        node.GID,
 				Size:       node.Size,
 				Mode:       node.Mode,
 				ModTime:    node.ModTime,
 				AccessTime: node.AccessTime,
 				ChangeTime: node.ChangeTime,
 				StructType: "node",
 			})
 			if err != nil {
 				Warnf("JSON encode failed: %v\n", err)
 			}
@@ -1,92 +0,0 @@
 package main
 import (
 	"bytes"
 	"encoding/json"
 	"os"
 	"testing"
 	"time"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
 func TestLsNodeJSON(t *testing.T) {
 	for _, c := range []struct {
 		path string
 		restic.Node
 		expect string
 	}{
 		// Mode is omitted when zero.
 		// Permissions, by convention is "-" per mode bit
 		{
 			path: "/bar/baz",
 			Node: restic.Node{
 				Name: "baz",
 				Type: "file",
 				Size: 12345,
 				UID:  10000000,
 				GID:  20000000,
 				User:  "nobody",
 				Group: "nobodies",
 				Links: 1,
 			},
 			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 		// Even empty files get an explicit size.
 		{
 			path: "/foo/empty",
 			Node: restic.Node{
 				Name: "empty",
 				Type: "file",
 				Size: 0,
 				UID:  1001,
 				GID:  1001,
 				User:  "not printed",
 				Group: "not printed",
 				Links: 0xF00,
 			},
 			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 		// Non-regular files do not get a size.
 		// Mode is printed in decimal, including the type bits.
 		{
 			path: "/foo/link",
 			Node: restic.Node{
 				Name:       "link",
 				Type:       "symlink",
 				Mode:       os.ModeSymlink | 0777,
 				LinkTarget: "not printed",
 			},
 			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 		},
 		{
 			path: "/some/directory",
 			Node: restic.Node{
 				Name:       "directory",
 				Type:       "dir",
 				Mode:       os.ModeDir | 0755,
 				ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
 				AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
 				ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
 			},
 			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
 		},
 	} {
 		buf := new(bytes.Buffer)
 		enc := json.NewEncoder(buf)
 		err := lsNodeJSON(enc, c.path, &c.Node)
 		rtest.OK(t, err)
 		rtest.Equals(t, c.expect+"\n", buf.String())
 		// Sanity check: output must be valid JSON.
 		var v interface{}
 		err = json.NewDecoder(buf).Decode(&v)
 		rtest.OK(t, err)
 	}
 }
@@ -122,7 +122,6 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
 		systemFuse.MaxReadahead(128 * 1024),
 	}
 	if opts.AllowOther {
@@ -162,8 +161,7 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	root := fuse.NewRoot(repo, cfg)
 	Printf("Now serving the repository at %s\n", mountpoint)
-	Printf("Use another terminal or tool to browse the contents of this folder.\n")
+	Printf("When finished, quit with Ctrl-c or umount the mountpoint.\n")
 	Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, root)
@@ -66,7 +66,6 @@ func addPruneOptions(c *cobra.Command) {
 }
 func verifyPruneOptions(opts *PruneOptions) error {
 	opts.MaxRepackBytes = math.MaxUint64
 	if len(opts.MaxRepackSize) > 0 {
 		size, err := parseSizeStr(opts.MaxRepackSize)
 		if err != nil {
@@ -120,6 +119,18 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 func shortenStatus(maxLength int, s string) string {
 	if len(s) <= maxLength {
 		return s
 	}
 	if maxLength < 3 {
 		return s[:maxLength]
 	}
 	return s[:maxLength-3] + "..."
 }
 func runPrune(opts PruneOptions, gopts GlobalOptions) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
@@ -238,7 +249,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			"Integrity check failed: Data seems to be missing.\n"+
 			"Will not start prune to prevent (additional) data loss!\n"+
 			"Please report this error (along with the output of the 'prune' run) at\n"+
-			"https://github.com/restic/restic/issues/new/choose\n", usedBlobs)
+			"https://github.com/restic/restic/issues/new/choose", usedBlobs)
 		return errorIndexIncomplete
 	}
@@ -313,7 +324,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 			// Pack size does not fit and pack is needed => error
 			// If the pack is not needed, this is no error, the pack can
 			// and will be simply removed, see below.
-			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n",
+			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.",
 				id.Str(), p.unusedSize+p.usedSize, packSize)
 			return errorSizeNotMatching
 		}
@@ -419,7 +430,11 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	for _, p := range repackCandidates {
 		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
-		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
+
 		reachedRepackSize := false
 		if opts.MaxRepackBytes > 0 {
 			reachedRepackSize = stats.size.repack+p.unusedSize+p.usedSize > opts.MaxRepackBytes
 		}
 		switch {
 		case reachedRepackSize:
@@ -459,7 +474,7 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
 	Verbosef("\nto repack:   %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
-	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
+	Verbosef("this removes %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
 	Verbosef("to delete:   %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
 	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
 	Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
@@ -73,27 +73,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
 		}
 	} else {
 		Verbosef("loading indexes...\n")
-		mi := repository.NewMasterIndex()
+		err := repo.LoadIndex(gopts.ctx)
 		err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
 			if err != nil {
 				Warnf("removing invalid index %v: %v\n", id, err)
 				obsoleteIndexes = append(obsoleteIndexes, id)
 				return nil
 			}
 			mi.Insert(idx)
 			return nil
 		})
 		if err != nil {
 			return err
 		}
 		err = mi.MergeFinalIndexes()
 		if err != nil {
 			return err
 		}
 		err = repo.SetIndex(mi)
 		if err != nil {
 			return err
 		}
@@ -1,7 +1,6 @@
 package main
 import (
 	"context"
 	"os"
 	"time"
@@ -12,11 +11,11 @@ import (
 var cmdRecover = &cobra.Command{
 	Use:   "recover [flags]",
-	Short: "Recover data from the repository not referenced by snapshots",
+	Short: "Recover data from the repository",
 	Long: `
 The "recover" command builds a new snapshot from all directories it can find in
-the raw data of the repository which are not referenced in an existing snapshot.
+the raw data of the repository. It can be used if, for example, a snapshot has
-It can be used if, for example, a snapshot has been removed by accident with "forget".
+been removed by accident with "forget".
 EXIT STATUS
 ===========
@@ -60,14 +59,24 @@ func runRecover(gopts GlobalOptions) error {
 	trees := make(map[restic.ID]bool)
 	for blob := range repo.Index().Each(gopts.ctx) {
-		if blob.Type == restic.TreeBlob {
+		if blob.Blob.Type != restic.TreeBlob {
 			continue
 		}
 		trees[blob.Blob.ID] = false
 	}
 	cur := 0
 	max := len(trees)
 	Verbosef("load %d trees\n\n", len(trees))
 	for id := range trees {
 		cur++
 		Verbosef("\rtree (%v/%v)", cur, max)
 		if !trees[id] {
 			trees[id] = false
 		}
 	Verbosef("load %d trees\n", len(trees))
 	bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
 		tree, err := repo.LoadTree(gopts.ctx, id)
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
@@ -75,39 +84,28 @@ func runRecover(gopts GlobalOptions) error {
 		}
 		for _, node := range tree.Nodes {
-			if node.Type == "dir" && node.Subtree != nil {
+			if node.Type != "dir" || node.Subtree == nil {
-				trees[*node.Subtree] = true
+				continue
 			}
 		}
 		bar.Add(1)
 	}
 	bar.Done()
-	Verbosef("load snapshots\n")
+			subtree := *node.Subtree
-	err = restic.ForAllSnapshots(gopts.ctx, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+			trees[subtree] = true
 		trees[*sn.Tree] = true
 		return nil
 	})
 	if err != nil {
 		return err
 		}
-	Verbosef("done\n")
+	}
 	Verbosef("\ndone\n")
 	roots := restic.NewIDSet()
 	for id, seen := range trees {
-		if !seen {
+		if seen {
-			Verboseff("found root tree %v\n", id.Str())
+			continue
 		}
 		roots.Insert(id)
 	}
 	}
 	Printf("\nfound %d unreferenced roots\n", len(roots))
-	if len(roots) == 0 {
+	Verbosef("found %d roots\n", len(roots))
 		Verbosef("no snapshot to write.\n")
 		return nil
 	}
-	tree := restic.NewTree(len(roots))
+	tree := restic.NewTree()
 	for id := range roots {
 		var subtreeID = id
 		node := restic.Node{
@@ -119,7 +117,7 @@ func runRecover(gopts GlobalOptions) error {
 			ModTime:    time.Now(),
 			ChangeTime: time.Now(),
 		}
-		err := tree.Insert(&node)
+		err = tree.Insert(&node)
 		if err != nil {
 			return err
 		}
@@ -135,23 +133,19 @@ func runRecover(gopts GlobalOptions) error {
 		return errors.Fatalf("unable to save blobs to the repo: %v", err)
 	}
-	return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
+	sn, err := restic.NewSnapshot([]string{"/recover"}, []string{}, hostname, time.Now())
 }
 func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
 	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
-	sn.Tree = tree
+	sn.Tree = &treeID
-	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+	id, err := repo.SaveJSONUnpacked(gopts.ctx, restic.SnapshotFile, sn)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 	Printf("saved new snapshot %v\n", id.Str())
 	return nil
 }
@@ -2,7 +2,6 @@ package main
 import (
 	"strings"
 	"time"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
@@ -118,7 +117,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts, nil)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
@@ -203,7 +202,6 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	if opts.Verify {
 		Verbosef("verifying files in %s\n", opts.Target)
 		var count int
 		t0 := time.Now()
 		count, err = res.VerifyFiles(ctx, opts.Target)
 		if err != nil {
 			return err
@@ -211,8 +209,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		if totalErrors > 0 {
 			return errors.Fatalf("There were %d errors\n", totalErrors)
 		}
-		Verbosef("finished verifying %d files in %s (took %s)\n", count, opts.Target,
+		Verbosef("finished verifying %d files in %s\n", count, opts.Target)
 			time.Since(t0).Round(time.Millisecond))
 	}
 	return nil
@@ -71,7 +71,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
 		}
 	}
-	Verbosef("writing restic to %v\n", opts.Output)
+	Printf("writing restic to %v\n", opts.Output)
 	v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef)
 	if err != nil {
@@ -36,8 +36,7 @@ type SnapshotOptions struct {
 	Tags    restic.TagLists
 	Paths   []string
 	Compact bool
-	Last    bool // This option should be removed in favour of Latest.
+	Last    bool
 	Latest  int
 	GroupBy string
 }
@@ -52,12 +51,6 @@ func init() {
 	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format")
 	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
 	err := f.MarkDeprecated("last", "use --latest 1")
 	if err != nil {
 		// MarkDeprecated only returns an error when the flag is not found
 		panic(err)
 	}
 	f.IntVar(&snapshotOptions.Latest, "latest", 0, "only show the last `n` snapshots for each host and path")
 	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
 }
@@ -89,11 +82,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	for k, list := range snapshotGroups {
 		if opts.Last {
-			// This branch should be removed in the same time
+			list = FilterLastSnapshots(list)
 			// that --last.
 			list = FilterLastestSnapshots(list, 1)
 		} else if opts.Latest > 0 {
 			list = FilterLastestSnapshots(list, opts.Latest)
 		}
 		sort.Sort(sort.Reverse(list))
 		snapshotGroups[k] = list
@@ -136,22 +125,21 @@ func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
 	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
 }
-// FilterLastestSnapshots filters a list of snapshots to only return
+// FilterLastSnapshots filters a list of snapshots to only return the last
-// the limit last entries for each hostname and path. If the snapshot
+// entry for each hostname and path. If the snapshot contains multiple paths,
-// contains multiple paths, they will be joined and treated as one
+// they will be joined and treated as one item.
-// item.
+func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
 func FilterLastestSnapshots(list restic.Snapshots, limit int) restic.Snapshots {
 	// Sort the snapshots so that the newer ones are listed first
 	sort.SliceStable(list, func(i, j int) bool {
 		return list[i].Time.After(list[j].Time)
 	})
 	var results restic.Snapshots
-	seen := make(map[filterLastSnapshotsKey]int)
+	seen := make(map[filterLastSnapshotsKey]bool)
 	for _, sn := range list {
 		key := newFilterLastSnapshotsKey(sn)
-		if seen[key] < limit {
+		if !seen[key] {
-			seen[key]++
+			seen[key] = true
 			results = append(results, sn)
 		}
 	}
@@ -25,21 +25,16 @@ const numDeleteWorkers = 8
 func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
 	totalCount := len(fileList)
 	fileChan := make(chan restic.ID)
-	wg, ctx := errgroup.WithContext(gopts.ctx)
+	go func() {
 	wg.Go(func() error {
 		defer close(fileChan)
 		for id := range fileList {
-			select {
+			fileChan <- id
 			case fileChan <- id:
 			case <-ctx.Done():
 				return ctx.Err()
 		}
-		}
+		close(fileChan)
-		return nil
+	}()
 	})
 	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted")
 	defer bar.Done()
 	wg, ctx := errgroup.WithContext(gopts.ctx)
 	for i := 0; i < numDeleteWorkers; i++ {
 		wg.Go(func() error {
 			for id := range fileChan {
@@ -23,7 +23,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			for _, s := range snapshotIDs {
 				if s == "latest" {
 					usedFilter = true
-					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts, nil)
+					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts)
 					if err != nil {
 						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
 						continue
@@ -31,7 +31,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 				} else {
 					id, err = restic.FindSnapshot(ctx, repo, s)
 					if err != nil {
-						Warnf("Ignoring %q: %v\n", s, err)
+						Warnf("Ignoring %q, it is not a snapshot id\n", s)
 						continue
 					}
 				}
@@ -31,7 +31,6 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
 	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/restic/restic/internal/errors"
@@ -40,7 +39,7 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
-var version = "0.13.0"
+var version = "0.12.0"
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -61,7 +60,6 @@ type GlobalOptions struct {
 	CacheDir        string
 	NoCache         bool
 	CACerts         []string
 	InsecureTLS     bool
 	TLSClientCert   string
 	CleanupCache    bool
@@ -98,8 +96,6 @@ func init() {
 	var cancel context.CancelFunc
 	globalOptions.ctx, cancel = context.WithCancel(context.Background())
 	AddCleanupHandler(func() error {
 		// Must be called before the unlock cleanup handler to ensure that the latter is
 		// not blocked due to limited number of backend connections, see #1434
 		cancel()
 		return nil
 	})
@@ -118,13 +114,10 @@ func init() {
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
 	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "`file` to load root certificates from (default: use system certificates)")
 	f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key")
 	f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repo (insecure)")
 	f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
 	f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
 	// Use our "generate" command instead of the cobra provided "completion" command
 	cmdRoot.CompletionOptions.DisableDefaultCmd = true
 	restoreTerminal()
 }
@@ -149,13 +142,7 @@ func stdinIsTerminal() bool {
 }
 func stdoutIsTerminal() bool {
-	// mintty on windows can use pipes which behave like a posix terminal,
+	return terminal.IsTerminal(int(os.Stdout.Fd()))
 	// but which are not a terminal handle
 	return terminal.IsTerminal(int(os.Stdout.Fd())) || stdoutCanUpdateStatus()
 }
 func stdoutCanUpdateStatus() bool {
 	return termstatus.CanUpdateStatus(os.Stdout.Fd())
 }
 func stdoutTerminalWidth() int {
@@ -172,7 +159,7 @@ func stdoutTerminalWidth() int {
 // program execution must revert changes to the terminal configuration itself.
 // The terminal configuration is only restored while reading a password.
 func restoreTerminal() {
-	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
+	if !stdoutIsTerminal() {
 		return
 	}
@@ -201,21 +188,16 @@ func restoreTerminal() {
 }
 // ClearLine creates a platform dependent string to clear the current
-// line, so it can be overwritten.
+// line, so it can be overwritten. ANSI sequences are not supported on
-//
+// current windows cmd shell.
-// w should be the terminal width, or 0 to let clearLine figure it out.
+func ClearLine() string {
-func clearLine(w int) string {
+	if runtime.GOOS == "windows" {
-	if runtime.GOOS != "windows" {
+		if w := stdoutTerminalWidth(); w > 0 {
-		return "\x1b[2K"
+			return strings.Repeat(" ", w-1) + "\r"
 		}
 	// ANSI sequences are not supported on Windows cmd shell.
 	if w <= 0 {
 		if w = stdoutTerminalWidth(); w <= 0 {
 		return ""
 	}
-	}
+	return "\x1b[2K"
 	return strings.Repeat(" ", w-1) + "\r"
 }
 // Printf writes the message to the configured stdout stream.
@@ -256,6 +238,31 @@ func Verboseff(format string, args ...interface{}) {
 	}
 }
 // PrintProgress wraps fmt.Printf to handle the difference in writing progress
 // information to terminals and non-terminal stdout
 func PrintProgress(format string, args ...interface{}) {
 	var (
 		message         string
 		carriageControl string
 	)
 	message = fmt.Sprintf(format, args...)
 	if !(strings.HasSuffix(message, "\r") || strings.HasSuffix(message, "\n")) {
 		if stdoutIsTerminal() {
 			carriageControl = "\r"
 		} else {
 			carriageControl = "\n"
 		}
 		message = fmt.Sprintf("%s%s", message, carriageControl)
 	}
 	if stdoutIsTerminal() {
 		message = fmt.Sprintf("%s%s", ClearLine(), message)
 	}
 	fmt.Print(message)
 }
 // Warnf writes the message to the configured stderr stream.
 func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
@@ -357,7 +364,7 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
 	}
 	if len(password) == 0 {
-		return "", errors.Fatal("an empty password is not a password")
+		return "", errors.New("an empty password is not a password")
 	}
 	return password, nil
@@ -485,7 +492,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		return s, nil
 	}
-	if c.Created && !opts.JSON && stdoutIsTerminal() {
+	if c.Created && !opts.JSON {
 		Verbosef("created new cache in %v\n", c.Base)
 	}
@@ -504,9 +511,8 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 	// cleanup old cache dirs if instructed to do so
 	if opts.CleanupCache {
-		if stdoutIsTerminal() && !opts.JSON {
+		Printf("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
-			Verbosef("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base)
+
 		}
 		for _, item := range oldCacheDirs {
 			dir := filepath.Join(c.Base, item.Name())
 			err = fs.RemoveAll(dir)
@@ -557,12 +563,6 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
 		}
 		if cfg.KeyID == "" && cfg.Secret != "" {
 			return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
 		} else if cfg.KeyID != "" && cfg.Secret == "" {
 			return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
 		}
 		if cfg.Region == "" {
 			cfg.Region = os.Getenv("AWS_DEFAULT_REGION")
 		}
@@ -682,7 +682,6 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
 		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -705,7 +704,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	case "azure":
 		be, err = azure.Open(cfg.(azure.Config), rt)
 	case "swift":
-		be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+		be, err = swift.Open(cfg.(swift.Config), rt)
 	case "b2":
 		be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
@@ -763,7 +762,6 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	tropts := backend.TransportOptions{
 		RootCertFilenames:        globalOptions.CACerts,
 		TLSClientCertKeyFilename: globalOptions.TLSClientCert,
 		InsecureTLS:              globalOptions.InsecureTLS,
 	}
 	rt, err := backend.Transport(tropts)
 	if err != nil {
@@ -782,7 +780,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "azure":
 		return azure.Create(cfg.(azure.Config), rt)
 	case "swift":
-		return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+		return swift.Open(cfg.(swift.Config), rt)
 	case "b2":
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
@@ -67,7 +67,7 @@ func isSymlink(fi os.FileInfo) bool {
 func sameModTime(fi1, fi2 os.FileInfo) bool {
 	switch runtime.GOOS {
-	case "darwin", "freebsd", "openbsd", "netbsd", "solaris":
+	case "darwin", "freebsd", "openbsd", "netbsd":
 		if isSymlink(fi1) && isSymlink(fi2) {
 			return true
 		}
@@ -15,7 +15,6 @@ import (
 	"regexp"
 	"runtime"
 	"strings"
 	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -159,11 +158,8 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
 	buf := bytes.NewBuffer(nil)
 	globalOptions.stdout = buf
 	oldStdout := gopts.stdout
 	gopts.stdout = buf
 	defer func() {
 		globalOptions.stdout = os.Stdout
 		gopts.stdout = oldStdout
 	}()
 	opts := DiffOptions{
@@ -349,57 +345,6 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 }
 func TestDryRunBackup(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 	testSetupBackupData(t, env)
 	opts := BackupOptions{}
 	dryOpts := BackupOptions{DryRun: true}
 	// dry run before first backup
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
 	snapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(snapshotIDs) == 0,
 		"expected no snapshot, got %v", snapshotIDs)
 	packIDs := testRunList(t, "packs", env.gopts)
 	rtest.Assert(t, len(packIDs) == 0,
 		"expected no data, got %v", snapshotIDs)
 	indexIDs := testRunList(t, "index", env.gopts)
 	rtest.Assert(t, len(indexIDs) == 0,
 		"expected no index, got %v", snapshotIDs)
 	// first backup
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 	snapshotIDs = testRunList(t, "snapshots", env.gopts)
 	packIDs = testRunList(t, "packs", env.gopts)
 	indexIDs = testRunList(t, "index", env.gopts)
 	// dry run between backups
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
 	snapshotIDsAfter := testRunList(t, "snapshots", env.gopts)
 	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
 	dataIDsAfter := testRunList(t, "packs", env.gopts)
 	rtest.Equals(t, packIDs, dataIDsAfter)
 	indexIDsAfter := testRunList(t, "index", env.gopts)
 	rtest.Equals(t, indexIDs, indexIDsAfter)
 	// second backup, implicit incremental
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 	snapshotIDs = testRunList(t, "snapshots", env.gopts)
 	packIDs = testRunList(t, "packs", env.gopts)
 	indexIDs = testRunList(t, "index", env.gopts)
 	// another dry run
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
 	snapshotIDsAfter = testRunList(t, "snapshots", env.gopts)
 	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
 	dataIDsAfter = testRunList(t, "packs", env.gopts)
 	rtest.Equals(t, packIDs, dataIDsAfter)
 	indexIDsAfter = testRunList(t, "index", env.gopts)
 	rtest.Equals(t, indexIDs, indexIDsAfter)
 }
 func TestBackupNonExistingFile(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -429,11 +374,10 @@ func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, rem
 	// Get all tree packs
 	rtest.OK(t, r.LoadIndex(gopts.ctx))
 	treePacks := restic.NewIDSet()
-	for pb := range r.Index().Each(context.TODO()) {
+	for _, idx := range r.Index().(*repository.MasterIndex).All() {
-		if pb.Type == restic.TreeBlob {
+		for _, id := range idx.TreePacks() {
-			treePacks.Insert(pb.PackID)
+			treePacks.Insert(id)
 		}
 	}
@@ -492,10 +436,11 @@ func TestBackupTreeLoadError(t *testing.T) {
 	r, err := OpenRepository(env.gopts)
 	rtest.OK(t, err)
 	rtest.OK(t, r.LoadIndex(env.gopts.ctx))
-	treePacks := restic.NewIDSet()
+	// collect tree packs of subdirectory
-	for pb := range r.Index().Each(context.TODO()) {
+	subTreePacks := restic.NewIDSet()
-		if pb.Type == restic.TreeBlob {
+	for _, idx := range r.Index().(*repository.MasterIndex).All() {
-			treePacks.Insert(pb.PackID)
+		for _, id := range idx.TreePacks() {
 			subTreePacks.Insert(id)
 		}
 	}
@@ -503,7 +448,7 @@ func TestBackupTreeLoadError(t *testing.T) {
 	testRunCheck(t, env.gopts)
 	// delete the subdirectory pack first
-	for id := range treePacks {
+	for id := range subTreePacks {
 		rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}))
 	}
 	testRunRebuildIndex(t, env.gopts)
@@ -854,25 +799,6 @@ func TestCopyIncremental(t *testing.T) {
 		len(copiedSnapshotIDs), len(snapshotIDs))
 }
 func TestCopyUnstableJSON(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 	env2, cleanup2 := withTestEnvironment(t)
 	defer cleanup2()
 	// contains a symlink created using `ln -s '../i/'$'\355\246\361''d/samba' broken-symlink`
 	datafile := filepath.Join("testdata", "copy-unstable-json.tar.gz")
 	rtest.SetupTarTestFixture(t, env.base, datafile)
 	testRunInit(t, env2.gopts)
 	testRunCopy(t, env.gopts, env2.gopts)
 	testRunCheck(t, env2.gopts)
 	copiedSnapshotIDs := testRunList(t, "snapshots", env2.gopts)
 	rtest.Assert(t, 1 == len(copiedSnapshotIDs), "still expected %v snapshot, found %v",
 		1, len(copiedSnapshotIDs))
 }
 func TestInitCopyChunkerParams(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -1088,41 +1014,6 @@ func TestKeyAddRemove(t *testing.T) {
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
 }
 type emptySaveBackend struct {
 	restic.Backend
 }
 func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
 	return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
 }
 func TestKeyProblems(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 	testRunInit(t, env.gopts)
 	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
 		return &emptySaveBackend{r}, nil
 	}
 	testKeyNewPassword = "geheim2"
 	defer func() {
 		testKeyNewPassword = ""
 	}()
 	err := runKey(env.gopts, []string{"passwd"})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected passwd change to fail")
 	err = runKey(env.gopts, []string{"add"})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected key adding to fail")
 	t.Logf("testing access with initial password %q\n", env.gopts.password)
 	rtest.OK(t, runKey(env.gopts, []string{"list"}))
 	testRunCheck(t, env.gopts)
 }
 func testFileSize(filename string, size int64) error {
 	fi, err := os.Stat(filename)
 	if err != nil {
@@ -1420,7 +1311,7 @@ func TestFindJSON(t *testing.T) {
 	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
 }
-func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
+func TestRebuildIndex(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -1440,10 +1331,8 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
 		t.Fatalf("did not find hint for rebuild-index command")
 	}
 	env.gopts.backendTestHook = backendTestHook
 	testRunRebuildIndex(t, env.gopts)
 	env.gopts.backendTestHook = nil
 	out, err = testRunCheckOutput(env.gopts)
 	if len(out) != 0 {
 		t.Fatalf("expected no output from the checker, got: %v", out)
@@ -1454,57 +1343,9 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
 	}
 }
 func TestRebuildIndex(t *testing.T) {
 	testRebuildIndex(t, nil)
 }
 func TestRebuildIndexAlwaysFull(t *testing.T) {
 	indexFull := repository.IndexFull
 	defer func() {
 		repository.IndexFull = indexFull
 	}()
 	repository.IndexFull = func(*repository.Index) bool { return true }
-	testRebuildIndex(t, nil)
+	TestRebuildIndex(t)
 }
 // indexErrorBackend modifies the first index after reading.
 type indexErrorBackend struct {
 	restic.Backend
 	lock     sync.Mutex
 	hasErred bool
 }
 func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
 	return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
 		// protect hasErred
 		b.lock.Lock()
 		defer b.lock.Unlock()
 		if !b.hasErred && h.Type == restic.IndexFile {
 			b.hasErred = true
 			return consumer(errorReadCloser{rd})
 		}
 		return consumer(rd)
 	})
 }
 type errorReadCloser struct {
 	io.Reader
 }
 func (erd errorReadCloser) Read(p []byte) (int, error) {
 	n, err := erd.Reader.Read(p)
 	if n > 0 {
 		p[0] ^= 1
 	}
 	return n, err
 }
 func TestRebuildIndexDamage(t *testing.T) {
 	testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
 		return &indexErrorBackend{
 			Backend: r,
 		}, nil
 	})
 }
 type appendOnlyBackend struct {
@@ -1747,7 +1588,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 // a listOnceBackend only allows listing once per filetype
 // listing filetypes more than once may cause problems with eventually consistent
-// backends (like e.g. Amazon S3) as the second listing may be inconsistent to what
+// backends (like e.g. AWS S3) as the second listing may be inconsistent to what
 // is expected by the first listing + some operations.
 type listOnceBackend struct {
 	restic.Backend
@@ -1975,8 +1816,10 @@ var diffOutputRegexPatterns = []string{
 	"Removed: +2[0-9]{2}\\.[0-9]{3} KiB",
 }
-func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
+func TestDiff(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 	testRunInit(t, env.gopts)
 	datadir := filepath.Join(env.base, "testdata")
@@ -2012,82 +1855,19 @@ func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	_, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 	return env, cleanup, firstSnapshotID, secondSnapshotID
 }
 func TestDiff(t *testing.T) {
 	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
 	defer cleanup()
 	// quiet suppresses the diff output except for the summary
 	env.gopts.Quiet = false
 	_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
 	rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
 	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	rtest.OK(t, err)
+	if err != nil {
 		t.Fatalf("expected no error from diff for test repository, got %v", err)
 	}
 	for _, pattern := range diffOutputRegexPatterns {
 		r, err := regexp.Compile(pattern)
 		rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern)
 		rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out)
 	}
 	// check quiet output
 	env.gopts.Quiet = true
 	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
 	rtest.OK(t, err)
 	rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
 }
 type typeSniffer struct {
 	MessageType string `json:"message_type"`
 }
 func TestDiffJSON(t *testing.T) {
 	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
 	defer cleanup()
 	// quiet suppresses the diff output except for the summary
 	env.gopts.Quiet = false
 	env.gopts.JSON = true
 	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
 	rtest.OK(t, err)
 	var stat DiffStatsContainer
 	var changes int
 	scanner := bufio.NewScanner(strings.NewReader(out))
 	for scanner.Scan() {
 		line := scanner.Text()
 		var sniffer typeSniffer
 		rtest.OK(t, json.Unmarshal([]byte(line), &sniffer))
 		switch sniffer.MessageType {
 		case "change":
 			changes++
 		case "statistics":
 			rtest.OK(t, json.Unmarshal([]byte(line), &stat))
 		default:
 			t.Fatalf("unexpected message type %v", sniffer.MessageType)
 		}
 	}
 	rtest.Equals(t, 9, changes)
 	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
 		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
 		stat.ChangedFiles == 1, "unexpected statistics")
 	// check quiet output
 	env.gopts.Quiet = true
 	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
 	rtest.OK(t, err)
 	stat = DiffStatsContainer{}
 	rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat))
 	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
 		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
 		stat.ChangedFiles == 1, "unexpected statistics")
 	rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids")
 }
 type writeToOnly struct {
@@ -16,7 +16,6 @@ var globalLocks struct {
 	cancelRefresh chan struct{}
 	refreshWG     sync.WaitGroup
 	sync.Mutex
 	sync.Once
 }
 func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
@@ -28,12 +27,6 @@ func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*resti
 }
 func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
 	// make sure that a repository is unlocked properly and after cancel() was
 	// called by the cleanup handler in global.go
 	globalLocks.Do(func() {
 		AddCleanupHandler(unlockAll)
 	})
 	lockFn := restic.NewLock
 	if exclusive {
 		lockFn = restic.NewExclusiveLock
@@ -135,3 +128,7 @@ func unlockAll() error {
 	return nil
 }
 func init() {
 	AddCleanupHandler(unlockAll)
 }
@@ -4,17 +4,15 @@ import (
 	"fmt"
 	"os"
 	"strconv"
 	"strings"
 	"time"
 	"github.com/restic/restic/internal/ui/progress"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 // calculateProgressInterval returns the interval configured via RESTIC_PROGRESS_FPS
 // or if unset returns an interval for 60fps on interactive terminals and 0 (=disabled)
-// for non-interactive terminals or when run using the --quiet flag
+// for non-interactive terminals
-func calculateProgressInterval(show bool, json bool) time.Duration {
+func calculateProgressInterval() time.Duration {
 	interval := time.Second / 60
 	fps, err := strconv.ParseFloat(os.Getenv("RESTIC_PROGRESS_FPS"), 64)
 	if err == nil && fps > 0 {
@@ -22,7 +20,7 @@ func calculateProgressInterval(show bool, json bool) time.Duration {
 			fps = 60
 		}
 		interval = time.Duration(float64(time.Second) / fps)
-	} else if !json && !stdoutCanUpdateStatus() || !show {
+	} else if !stdoutIsTerminal() {
 		interval = 0
 	}
 	return interval
@@ -33,8 +31,7 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 	if !show {
 		return nil
 	}
-	interval := calculateProgressInterval(show, false)
+	interval := calculateProgressInterval()
 	canUpdateStatus := stdoutCanUpdateStatus()
 	return progress.New(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
 		var status string
@@ -45,36 +42,13 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
 				formatDuration(d), formatPercent(v, max), v, max, description)
 		}
-		printProgress(status, canUpdateStatus)
+		if w := stdoutTerminalWidth(); w > 0 {
 			status = shortenStatus(w, status)
 		}
 		PrintProgress("%s", status)
 		if final {
 			fmt.Print("\n")
 		}
 	})
 }
 func printProgress(status string, canUpdateStatus bool) {
 	w := stdoutTerminalWidth()
 	if w > 0 {
 		if w < 3 {
 			status = termstatus.Truncate(status, w)
 		} else {
 			status = termstatus.Truncate(status, w-3) + "..."
 		}
 	}
 	var carriageControl, clear string
 	if canUpdateStatus {
 		clear = clearLine(w)
 	}
 	if !(strings.HasSuffix(status, "\r") || strings.HasSuffix(status, "\n")) {
 		if canUpdateStatus {
 			carriageControl = "\r"
 		} else {
 			carriageControl = "\n"
 		}
 	}
 	_, _ = os.Stdout.Write([]byte(clear + status + carriageControl))
 }
@@ -9,7 +9,6 @@ import (
 type secondaryRepoOptions struct {
 	Repo            string
 	RepositoryFile  string
 	password        string
 	PasswordFile    string
 	PasswordCommand string
@@ -18,25 +17,18 @@ type secondaryRepoOptions struct {
 func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repoPrefix string, repoUsage string) {
 	f.StringVarP(&opts.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
 	f.StringVarP(&opts.RepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
 	f.StringVarP(&opts.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
 	f.StringVarP(&opts.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
 	f.StringVarP(&opts.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
 }
 func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, error) {
-	if opts.Repo == "" && opts.RepositoryFile == "" {
+	if opts.Repo == "" {
-		return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2 or --repository-file2)")
+		return GlobalOptions{}, errors.Fatal("Please specify a " + repoPrefix + " repository location (--repo2)")
 	}
 	if opts.Repo != "" && opts.RepositoryFile != "" {
 		return GlobalOptions{}, errors.Fatal("Options --repo2 and --repository-file2 are mutually exclusive, please specify only one")
 	}
 	var err error
 	dstGopts := gopts
 	dstGopts.Repo = opts.Repo
 	dstGopts.RepositoryFile = opts.RepositoryFile
 	dstGopts.PasswordFile = opts.PasswordFile
 	dstGopts.PasswordCommand = opts.PasswordCommand
 	dstGopts.KeyHint = opts.KeyHint
@@ -1,132 +0,0 @@
 package main
 import (
 	"io/ioutil"
 	"path/filepath"
 	"testing"
 	rtest "github.com/restic/restic/internal/test"
 )
 //TestFillSecondaryGlobalOpts tests valid and invalid data on fillSecondaryGlobalOpts-function
 func TestFillSecondaryGlobalOpts(t *testing.T) {
 	//secondaryRepoTestCase defines a struct for test cases
 	type secondaryRepoTestCase struct {
 		Opts     secondaryRepoOptions
 		DstGOpts GlobalOptions
 	}
 	//validSecondaryRepoTestCases is a list with test cases that must pass
 	var validSecondaryRepoTestCases = []secondaryRepoTestCase{
 		{
 			// Test if Repo and Password are parsed correctly.
 			Opts: secondaryRepoOptions{
 				Repo:     "backupDst",
 				password: "secretDst",
 			},
 			DstGOpts: GlobalOptions{
 				Repo:     "backupDst",
 				password: "secretDst",
 			},
 		},
 		{
 			// Test if RepositoryFile and PasswordFile are parsed correctly.
 			Opts: secondaryRepoOptions{
 				RepositoryFile: "backupDst",
 				PasswordFile:   "passwordFileDst",
 			},
 			DstGOpts: GlobalOptions{
 				RepositoryFile: "backupDst",
 				password:       "secretDst",
 				PasswordFile:   "passwordFileDst",
 			},
 		},
 		{
 			// Test if RepositoryFile and PasswordCommand are parsed correctly.
 			Opts: secondaryRepoOptions{
 				RepositoryFile:  "backupDst",
 				PasswordCommand: "echo secretDst",
 			},
 			DstGOpts: GlobalOptions{
 				RepositoryFile:  "backupDst",
 				password:        "secretDst",
 				PasswordCommand: "echo secretDst",
 			},
 		},
 	}
 	//invalidSecondaryRepoTestCases is a list with test cases that must fail
 	var invalidSecondaryRepoTestCases = []secondaryRepoTestCase{
 		{
 			// Test must fail on no repo given.
 			Opts: secondaryRepoOptions{},
 		},
 		{
 			// Test must fail as Repo and RepositoryFile are both given
 			Opts: secondaryRepoOptions{
 				Repo:           "backupDst",
 				RepositoryFile: "backupDst",
 			},
 		},
 		{
 			// Test must fail as PasswordFile and PasswordCommand are both given
 			Opts: secondaryRepoOptions{
 				Repo:            "backupDst",
 				PasswordFile:    "passwordFileDst",
 				PasswordCommand: "notEmpty",
 			},
 		},
 		{
 			// Test must fail as PasswordFile does not exist
 			Opts: secondaryRepoOptions{
 				Repo:         "backupDst",
 				PasswordFile: "NonExistingFile",
 			},
 		},
 		{
 			// Test must fail as PasswordCommand does not exist
 			Opts: secondaryRepoOptions{
 				Repo:            "backupDst",
 				PasswordCommand: "notEmpty",
 			},
 		},
 		{
 			// Test must fail as no password is given.
 			Opts: secondaryRepoOptions{
 				Repo: "backupDst",
 			},
 		},
 	}
 	//gOpts defines the Global options used in the secondary repository tests
 	var gOpts = GlobalOptions{
 		Repo:           "backupSrc",
 		RepositoryFile: "backupSrc",
 		password:       "secretSrc",
 		PasswordFile:   "passwordFileSrc",
 	}
 	//Create temp dir to create password file.
 	dir, cleanup := rtest.TempDir(t)
 	defer cleanup()
 	cleanup = rtest.Chdir(t, dir)
 	defer cleanup()
 	//Create temporary password file
 	err := ioutil.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
 	rtest.OK(t, err)
 	// Test all valid cases
 	for _, testCase := range validSecondaryRepoTestCases {
 		DstGOpts, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
 		rtest.OK(t, err)
 		rtest.Equals(t, DstGOpts, testCase.DstGOpts)
 	}
 	// Test all invalid cases
 	for _, testCase := range invalidSecondaryRepoTestCases {
 		_, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
 		rtest.Assert(t, err != nil, "Expected error, but function did not return an error")
 	}
 }
Author	SHA1	Message	Date
Michael Eischer	e1bb2129ad	report result of delete operation after failed upload	2021-03-08 21:54:46 +01:00
Michael Eischer	95b7f8dd81	s3: Fix sanity check The sanity check shouldn't replace the error message if there is already one.	2021-03-08 21:54:46 +01:00
Michael Eischer	29e39e247a	stat file after retrying an upload	2021-03-08 21:54:46 +01:00