build(deps): bump docker/setup-buildx-action from 3 to 4

Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3 to 4.
- [Release notes](https://github.com/docker/setup-buildx-action/releases)
- [Commits](https://github.com/docker/setup-buildx-action/compare/v3...v4)

---
updated-dependencies:
- dependency-name: docker/setup-buildx-action
  dependency-version: '4'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

.dockerignore

@@ -0,0 +1,12 @@
+# Actual layer caching is impossible due to .git, but
+# that must be included for provenance reasons.  These ignores
+# are strictly for hygenic build.
+*
+!/*.go
+!/go.*
+!/cmd/*
+!/docker/entrypoint.sh
+!/internal/*
+!/helpers/*
+!/VERSION
+!/.git/

.github/PULL_REQUEST_TEMPLATE.md

@@ -28,13 +28,15 @@ Checklist
 You do not need to check all the boxes below all at once. Feel free to take
 your time and add more commits. If you're done and ready for review, please
 check the last box. Enable a checkbox by replacing [ ] with [x].
+
+Please always follow these steps:
+- Read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
+- Enable [maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
+- Run `gofmt` on the code in all commits.
+- Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
 -->
 
-- [ ] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
+- [ ] I have added tests for all code changes, see [writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests)
-- [ ] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
-- [ ] I have added tests for all code changes.
 - [ ] I have added documentation for relevant changes (in the manual).
 - [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
-- [ ] I have run `gofmt` on the code in all commits.
-- [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
 - [ ] I'm done! This pull request is ready for review.

.github/dependabot.yml

@@ -5,6 +5,10 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "monthly"
+    groups:
+      golang-x-deps:
+        patterns:
+          - "golang.org/x/*"
 
   # Dependencies listed in .github/workflows/*.yml
   - package-ecosystem: "github-actions"

.github/workflows/docker.yml

@@ -20,12 +20,16 @@ jobs:
       contents: read
       packages: write
 
+    outputs:
+      image: ${{ steps.image.outputs.image }}
+      digest: ${{ steps.build-and-push.outputs.digest }}
+
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Log in to the Container registry
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -37,6 +41,7 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
+            type=sha
             type=ref,event=branch
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
@@ -45,7 +50,7 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
 
       - name: Ensure consistent binaries
         run: |
@@ -55,6 +60,7 @@ jobs:
         if: github.ref != 'refs/heads/master'
 
       - name: Build and push Docker image
+        id: build-and-push
         uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1
         with:
           push: true
@@ -64,3 +70,26 @@ jobs:
           pull: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+        
+      - name: Output image
+        id: image
+        run: |
+          # NOTE: Set the image as an output because the `env` context is not
+          # available to the inputs of a reusable workflow call.
+          image_name="${REGISTRY}/${IMAGE_NAME}"
+          echo "image=$image_name" >> "$GITHUB_OUTPUT"
+        
+  provenance:
+    needs: [build-and-push-image]
+    permissions:
+      actions: read # for detecting the Github Actions environment.
+      id-token: write # for creating OIDC tokens for signing.
+      packages: write # for uploading attestations.
+    if: github.repository == 'restic/restic'
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0
+    with:
+      image: ${{ needs.build-and-push-image.outputs.image }}
+      digest: ${{ needs.build-and-push-image.outputs.digest }}
+      registry-username: ${{ github.actor }}
+    secrets:
+      registry-password: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 env:
-  latest_go: "1.22.x"
+  latest_go: "1.25.x"
   GO111MODULE: on
 
 jobs:
@@ -23,39 +23,29 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.22.x
+            go: 1.25.x
             os: windows-latest
 
           - job_name: macOS
-            go: 1.22.x
+            go: 1.25.x
             os: macOS-latest
             test_fuse: false
 
           - job_name: Linux
-            go: 1.22.x
+            go: 1.25.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
 
           - job_name: Linux (race)
-            go: 1.22.x
+            go: 1.25.x
             os: ubuntu-latest
             test_fuse: true
             test_opts: "-race"
 
           - job_name: Linux
-            go: 1.21.x
+            go: 1.24.x
-            os: ubuntu-latest
-            test_fuse: true
-
-          - job_name: Linux
-            go: 1.20.x
-            os: ubuntu-latest
-            test_fuse: true
-
-          - job_name: Linux
-            go: 1.19.x
             os: ubuntu-latest
             test_fuse: true
 
@@ -67,10 +57,10 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ matrix.go }}
 
@@ -195,7 +185,7 @@ jobs:
           # prepare credentials for Google Cloud Storage tests in a temp file
           export GOOGLE_APPLICATION_CREDENTIALS=$(mktemp --tmpdir restic-gcs-auth-XXXXXXX)
           echo $RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
-          go test -cover -parallel 4 ./internal/backend/...
+          go test -cover -parallel 5 -timeout 15m ./internal/backend/...
 
         # only run cloud backend tests for pull requests from and pushes to our
         # own repo, otherwise the secrets are not available
@@ -214,7 +204,6 @@ jobs:
 
   cross_compile:
     strategy:
-
       matrix:
         # run cross-compile in three batches parallel so the overall tests run faster
         subset:
@@ -231,10 +220,10 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ env.latest_go }}
 
@@ -253,18 +242,18 @@ jobs:
       checks: write
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Set up Go ${{ env.latest_go }}
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ${{ env.latest_go }}
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v9
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.57.1
+          version: v2.4.0
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
@@ -298,7 +287,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Docker meta
         id: meta
@@ -321,7 +310,7 @@ jobs:
         uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Build and push
         id: docker_build

.golangci.yml

@@ -1,69 +1,95 @@
-# This is the configuration for golangci-lint for the restic project.
+version: "2"
-#
-# A sample config with all settings is here:
-# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
-
 linters:
   # only enable the linters listed below
-  disable-all: true
+  default: none
   enable:
+    - asciicheck
+    # ensure that http response bodies are closed
+    - bodyclose
+    # restrict imports from other restic packages for internal/backend (cache exempt)
+    - depguard
+    - copyloopvar
     # make sure all errors returned by functions are handled
     - errcheck
-
-    # show how code can be simplified
-    - gosimple
-
-    # make sure code is formatted
-    - gofmt
-
     # examine code and report suspicious constructs, such as Printf calls whose
     # arguments do not align with the format string
     - govet
-
+    # consistent imports
-    # make sure names and comments are used according to the conventions
+    - importas
-    - revive
-
     # detect when assignments to existing variables are not used
     - ineffassign
-
+    - nolintlint
+    # make sure names and comments are used according to the conventions
+    - revive
     # run static analysis and find errors
     - staticcheck
-
     # find unused variables, functions, structs, types, etc.
     - unused
-
+  settings:
-    # parse and typecheck code
+    depguard:
-    - typecheck
+      rules:
-
+        # Prevent backend packages from importing the internal/restic package to keep the architectural layers intact.
-    # ensure that http response bodies are closed
+        backend-imports:
-    - bodyclose
+          files:
-
+            - "**/internal/backend/**"
-    - importas
+            - "!**/internal/backend/cache/**"
-
+            - "!**/internal/backend/test/**"
-issues:
+            - "!**/*_test.go"
-  # don't use the default exclude rules, this hides (among others) ignored
+          deny:
-  # errors from Close() calls
+            - pkg: "github.com/restic/restic/internal/restic"
-  exclude-use-default: false
+              desc: "internal/restic should not be imported to keep the architectural layers intact"
-
+            - pkg: "github.com/restic/restic/internal/repository"
-  # list of things to not warn about
+              desc: "internal/repository should not be imported to keep the architectural layers intact"
-  exclude:
+    importas:
-    # revive: do not warn about missing comments for exported stuff
+      alias:
-    - exported (function|method|var|type|const) .* should have comment or be unexported
+        - pkg: github.com/restic/restic/internal/test
-    # revive: ignore constants in all caps
+          alias: rtest
-    - don't use ALL_CAPS in Go names; use CamelCase
+    staticcheck:
-    # revive: lots of packages don't have such a comment
+      checks:
-    - "package-comments: should have a package comment"
+        # default
-    # staticcheck: there's no easy way to replace these packages
+        - "all"
-    - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
+        - "-ST1000"
-    - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
+        - "-ST1003"
-
+        - "-ST1016"
-  exclude-rules:
+        - "-ST1020"
-    # revive: ignore unused parameters in tests
+        - "-ST1021"
-    - path: (_test\.go|testing\.go|backend/.*/tests\.go)
+        - "-ST1022"
-      text: "unused-parameter:"
+        # extra disables
-
+        - "-QF1008" # don't warn about specifing name of embedded field on access
-linters-settings:
+  exclusions:
-  importas:
+    rules:
-    alias:
+      # revive: ignore unused parameters in tests
-      - pkg: github.com/restic/restic/internal/test
+      - path: (_test\.go|testing\.go|backend/.*/tests\.go)
-        alias: rtest
+        text: "unused-parameter:"
+      # revive: do not warn about missing comments for exported stuff
+      - path: (.+)\.go$
+        text: exported (function|method|var|type|const) .* should have comment or be unexported
+      # revive: ignore constants in all caps
+      - path: (.+)\.go$
+        text: don't use ALL_CAPS in Go names; use CamelCase
+      # revive: lots of packages don't have such a comment
+      - path: (.+)\.go$
+        text: "package-comments: should have a package comment"
+      # staticcheck: there's no easy way to replace these packages
+      - path: (.+)\.go$
+        text: 'SA1019: "golang.org/x/crypto/poly1305" is deprecated'
+      - path: (.+)\.go$
+        text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated'
+      - path: (.+)\.go$
+        text: "redefines-builtin-id:"
+      # revive: collection of helpers to implement a backend, more descriptive names would be too repetitive
+      - path: internal/backend/util/.*.go$
+        text: "var-naming: avoid meaningless package names"
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    # make sure code is formatted
+    - gofmt
+  exclusions:
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Table of Contents
 
+* [Changelog for 0.18.1](#changelog-for-restic-0181-2025-09-21)
+* [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
 * [Changelog for 0.17.3](#changelog-for-restic-0173-2024-11-08)
 * [Changelog for 0.17.2](#changelog-for-restic-0172-2024-10-27)
 * [Changelog for 0.17.1](#changelog-for-restic-0171-2024-09-05)
@@ -38,6 +40,485 @@
 * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
 
 
+# Changelog for restic 0.18.1 (2025-09-21)
+The following sections list the changes in restic 0.18.1 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Fix #5324: Correctly handle `backup --stdin-filename` with directory paths
+ * Fix #5325: Accept `RESTIC_HOST` environment variable in `forget` command
+ * Fix #5342: Ignore "chmod not supported" errors when writing files
+ * Fix #5344: Ignore `EOPNOTSUPP` errors for extended attributes
+ * Fix #5421: Fix rare crash if directory is removed during backup
+ * Fix #5429: Stop retrying uploads when rest-server runs out of space
+ * Fix #5467: Improve handling of download retries in `check` command
+
+## Details
+
+ * Bugfix #5324: Correctly handle `backup --stdin-filename` with directory paths
+
+   In restic 0.18.0, the `backup` command failed if a filename that includes at
+   least a directory was passed to `--stdin-filename`. For example,
+   `--stdin-filename /foo/bar` resulted in the following error:
+
+   ```
+   Fatal: unable to save snapshot: open /foo: no such file or directory
+   ```
+
+   This has now been fixed.
+
+   https://github.com/restic/restic/issues/5324
+   https://github.com/restic/restic/pull/5356
+
+ * Bugfix #5325: Accept `RESTIC_HOST` environment variable in `forget` command
+
+   The `forget` command did not use the host name from the `RESTIC_HOST`
+   environment variable when filtering snapshots. This has now been fixed.
+
+   https://github.com/restic/restic/issues/5325
+   https://github.com/restic/restic/pull/5327
+
+ * Bugfix #5342: Ignore "chmod not supported" errors when writing files
+
+   Restic 0.18.0 introduced a bug that caused `chmod xxx: operation not supported`
+   errors to appear when writing to a local file repository that did not support
+   chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors.
+
+   https://github.com/restic/restic/issues/5342
+
+ * Bugfix #5344: Ignore `EOPNOTSUPP` errors for extended attributes
+
+   Restic 0.18.0 added extended attribute support for NetBSD 10+, but not all
+   NetBSD filesystems support extended attributes. Other BSD systems can likewise
+   return `EOPNOTSUPP`, so restic now ignores these errors.
+
+   https://github.com/restic/restic/issues/5344
+
+ * Bugfix #5421: Fix rare crash if directory is removed during backup
+
+   In restic 0.18.0, the `backup` command could crash if a directory was removed
+   between reading its metadata and listing its directory content. This has now
+   been fixed.
+
+   https://github.com/restic/restic/pull/5421
+
+ * Bugfix #5429: Stop retrying uploads when rest-server runs out of space
+
+   When rest-server returns a `507 Insufficient Storage` error, it indicates that
+   no more storage capacity is available. Restic now correctly stops retrying
+   uploads in this case.
+
+   https://github.com/restic/restic/issues/5429
+   https://github.com/restic/restic/pull/5452
+
+ * Bugfix #5467: Improve handling of download retries in `check` command
+
+   In very rare cases, the `check` command could unnecessarily report repository
+   damage if the backend returned incomplete, corrupted data on the first download
+   try which is afterwards resolved by a download retry.
+
+   This could result in an error output like the following:
+
+   ```
+   Load(<data/34567890ab>, 33918928, 0) returned error, retrying after 871.35598ms: readFull: unexpected EOF
+   Load(<data/34567890ab>, 33918928, 0) operation successful after 1 retries
+   check successful on second attempt, original error pack 34567890ab[...] contains 6 errors: [blob 12345678[...]: decrypting blob <data/12345678> from 34567890 failed: ciphertext verification failed ...]
+   [...]
+   Fatal: repository contains errors
+   ```
+
+   This fix only applies to a very specific case where the log shows `operation
+   successful after 1 retries` followed by a `check successful on second attempt,
+   original error` that only reports `ciphertext verification failed` errors in the
+   pack file. If any other errors are reported in the pack file, then the
+   repository still has to be considered as damaged.
+
+   Now, only the check result of the last download retry is reported as intended.
+
+   https://github.com/restic/restic/issues/5467
+   https://github.com/restic/restic/pull/5495
+
+
+# Changelog for restic 0.18.0 (2025-03-27)
+The following sections list the changes in restic 0.18.0 relevant to
+restic users. The changes are ordered by importance.
+
+## Summary
+
+ * Sec #5291: Mitigate attack on content-defined chunking algorithm
+ * Fix #1843: Correctly restore long filepaths' timestamp on old Windows
+ * Fix #2165: Ignore disappeared backup source files
+ * Fix #5153: Include root tree when searching using `find --tree`
+ * Fix #5169: Prevent Windows VSS event log 8194 warnings for backup with fs snapshot
+ * Fix #5212: Fix duplicate data handling in `prune --max-unused`
+ * Fix #5249: Fix creation of oversized index by `repair index --read-all-packs`
+ * Fix #5259: Fix rare crash in command output
+ * Chg #4938: Update dependencies and require Go 1.23 or newer
+ * Chg #5162: Graduate feature flags
+ * Enh #1378: Add JSON support to `check` command
+ * Enh #2511: Support generating shell completions to stdout
+ * Enh #3697: Allow excluding online-only cloud files (e.g. OneDrive)
+ * Enh #4179: Add `sort` option to `ls` command
+ * Enh #4433: Change default sort order for `find` output
+ * Enh #4521: Add support for Microsoft Blob Storage access tiers
+ * Enh #4942: Add snapshot summary statistics to rewritten snapshots
+ * Enh #4948: Format exit errors as JSON when requested
+ * Enh #4983: Add SLSA provenance to GHCR container images
+ * Enh #5054: Enable compression for ZIP archives in `dump` command
+ * Enh #5081: Add retry mechanism for loading repository config
+ * Enh #5089: Allow including/excluding extended file attributes during `restore`
+ * Enh #5092: Show count of deleted files and directories during `restore`
+ * Enh #5109: Make small pack size configurable for `prune`
+ * Enh #5119: Add start and end timestamps to `backup` JSON output
+ * Enh #5131: Add DragonFlyBSD support
+ * Enh #5137: Make `tag` command print which snapshots were modified
+ * Enh #5141: Provide clear error message if AZURE_ACCOUNT_NAME is not set
+ * Enh #5173: Add experimental S3 cold storage support
+ * Enh #5174: Add xattr support for NetBSD 10+
+ * Enh #5251: Improve retry handling for flaky `rclone` backends
+ * Enh #5287: Make `recover` automatically rebuild index when needed
+
+## Details
+
+ * Security #5291: Mitigate attack on content-defined chunking algorithm
+
+   Restic uses [Rabin
+   Fingerprints](https://restic.net/blog/2015-09-12/restic-foundation1-cdc/) for
+   its content-defined chunker. The algorithm relies on a secret polynomial to
+   split files into chunks.
+
+   As shown in the paper "[Chunking Attacks on File Backup Services using
+   Content-Defined Chunking](https://eprint.iacr.org/2025/532.pdf)" by Boris
+   Alexeev, Colin Percival and Yan X Zhang, an attacker that can observe chunk
+   sizes for a known file can derive the secret polynomial. Knowledge of the
+   polynomial might in some cases allow an attacker to check whether certain large
+   files are stored in a repository.
+
+   A practical attack is nevertheless hard as restic merges multiple chunks into
+   opaque pack files and by default processes multiple files in parallel. This
+   likely prevents an attacker from matching pack files to the attacker-known file
+   and thereby prevents the attack.
+
+   Despite the low chances of a practical attack, restic now has added mitigation
+   that randomizes how chunks are assembled into pack files. This prevents
+   attackers from guessing which chunks are part of a pack file and thereby
+   prevents learning the chunk sizes.
+
+   https://github.com/restic/restic/issues/5291
+   https://github.com/restic/restic/pull/5295
+
+ * Bugfix #1843: Correctly restore long filepaths' timestamp on old Windows
+
+   The `restore` command now correctly restores timestamps for files with paths
+   longer than 256 characters on Windows versions prior to Windows 10 1607.
+
+   https://github.com/restic/restic/issues/1843
+   https://github.com/restic/restic/pull/5061
+
+ * Bugfix #2165: Ignore disappeared backup source files
+
+   The `backup` command now quietly skips files that are removed between directory
+   listing and backup, instead of printing errors like:
+
+   ```
+   error: lstat /some/file/name: no such file or directory
+   ```
+
+   https://github.com/restic/restic/issues/2165
+   https://github.com/restic/restic/issues/3098
+   https://github.com/restic/restic/pull/5143
+   https://github.com/restic/restic/pull/5145
+
+ * Bugfix #5153: Include root tree when searching using `find --tree`
+
+   The `restic find --tree` command did not find trees referenced by `restic
+   snapshot --json`. It now correctly includes the root tree when searching.
+
+   https://github.com/restic/restic/pull/5153
+
+ * Bugfix #5169: Prevent Windows VSS event log 8194 warnings for backup with fs snapshot
+
+   When running `backup` with the `--use-fs-snapshot` option in Windows with admin
+   rights, event logs like
+
+   ```
+   Volume Shadow Copy Service error: Unexpected error querying for the IVssWriterCallback interface.  hr = 0x80070005, Access is denied.
+   . This is often caused by incorrect security settings in either the writer or requester process. 
+   
+   Operation:
+      Gathering Writer Data
+   
+   Context:
+      Writer Class Id: {e8132975-6f93-4464-a53e-1050253ae220}
+      Writer Name: System Writer
+      Writer Instance ID: {54b151ac-d27d-4628-9cb0-2bc40959f50f}
+   ```
+
+   Are created several times even though the backup itself succeeds. This has now
+   been fixed.
+
+   https://github.com/restic/restic/issues/5169
+   https://github.com/restic/restic/pull/5170
+   https://forum.restic.net/t/windows-shadow-copy-snapshot-vss-unexpected-provider-error/3674/2
+
+ * Bugfix #5212: Fix duplicate data handling in `prune --max-unused`
+
+   The `prune --max-unused size` command did not correctly account for duplicate
+   data. If a repository contained a large amount of duplicate data, this could
+   previously result in pruning too little data. This has now been fixed.
+
+   https://github.com/restic/restic/pull/5212
+   https://forum.restic.net/t/restic-not-obeying-max-unused-parameter-on-prune/8879
+
+ * Bugfix #5249: Fix creation of oversized index by `repair index --read-all-packs`
+
+   Since restic 0.17.0, the new index created by `repair index --read-all-packs`
+   was written as a single large index. This significantly increased memory usage
+   while loading the index.
+
+   The index is now correctly split into multiple smaller indexes, and `repair
+   index` now also automatically splits oversized indexes.
+
+   https://github.com/restic/restic/pull/5249
+
+ * Bugfix #5259: Fix rare crash in command output
+
+   Some commands could in rare cases crash when trying to print status messages and
+   request retries at the same time, resulting in an error like the following:
+
+   ```
+   panic: runtime error: slice bounds out of range [468:156]
+   [...]
+   github.com/restic/restic/internal/ui/termstatus.(*lineWriter).Write(...)
+   	/restic/internal/ui/termstatus/stdio_wrapper.go:36 +0x136
+   ```
+
+   This has now been fixed.
+
+   https://github.com/restic/restic/issues/5259
+   https://github.com/restic/restic/pull/5300
+
+ * Change #4938: Update dependencies and require Go 1.23 or newer
+
+   We have updated all dependencies. Restic now requires Go 1.23 or newer to build.
+
+   This also disables support for TLS versions older than TLS 1.2. On Windows,
+   restic now requires at least Windows 10 or Windows Server 2016. On macOS, restic
+   now requires at least macOS 11 Big Sur.
+
+   https://github.com/restic/restic/pull/4938
+
+ * Change #5162: Graduate feature flags
+
+   The `deprecate-legacy-index`, `deprecate-s3-legacy-layout`,
+   `explicit-s3-anonymous-auth` and `safe-forget-keep-tags` features are now stable
+   and can no longer be disabled. The corresponding feature flags will be removed
+   in restic 0.19.0.
+
+   https://github.com/restic/restic/pull/5162
+
+ * Enhancement #1378: Add JSON support to `check` command
+
+   The `check` command now supports the `--json` option to output all statistics in
+   JSON format.
+
+   https://github.com/restic/restic/issues/1378
+   https://github.com/restic/restic/pull/5194
+
+ * Enhancement #2511: Support generating shell completions to stdout
+
+   The `generate` command now supports using `-` as the filename with the
+   `--[shell]-completion` option to write the generated output to stdout.
+
+   https://github.com/restic/restic/issues/2511
+   https://github.com/restic/restic/pull/5053
+
+ * Enhancement #3697: Allow excluding online-only cloud files (e.g. OneDrive)
+
+   Restic treated files synced using OneDrive Files On-Demand as though they were
+   regular files. This caused issues with VSS and could cause OneDrive to download
+   all files.
+
+   Restic now allows the user to exclude these files when backing up with the
+   `--exclude-cloud-files` option.
+
+   https://github.com/restic/restic/issues/3697
+   https://github.com/restic/restic/issues/4935
+   https://github.com/restic/restic/pull/4990
+
+ * Enhancement #4179: Add `sort` option to `ls` command
+
+   The `ls -l` command output can now be sorted using the new `--sort <field>`
+   option for the fields `name`, `size`, `time` (same as `mtime`), `mtime`,
+   `atime`, `ctime` and `extension`. A `--reverse` option is also available.
+
+   https://github.com/restic/restic/issues/4179
+   https://github.com/restic/restic/pull/5182
+
+ * Enhancement #4433: Change default sort order for `find` output
+
+   The `find` command now sorts snapshots from newest to oldest by default. The
+   previous oldest-to-newest order can be restored using the new `--reverse`
+   option.
+
+   https://github.com/restic/restic/issues/4433
+   https://github.com/restic/restic/pull/5184
+
+ * Enhancement #4521: Add support for Microsoft Blob Storage access tiers
+
+   The new `-o azure.access-tier=<tier>` option allows specifying the access tier
+   (`Hot`, `Cool` or `Cold`) for objects created in Microsoft Blob Storage. If
+   unspecified, the storage account's default tier is used.
+
+   There is no official `Archive` storage support in restic, use this option at
+   your own risk. To restore any data, it is necessary to manually warm up the
+   required data in the `Archive` tier.
+
+   https://github.com/restic/restic/issues/4521
+   https://github.com/restic/restic/pull/5046
+
+ * Enhancement #4942: Add snapshot summary statistics to rewritten snapshots
+
+   The `rewrite` command now supports a `--snapshot-summary` option to add
+   statistics data to snapshots. Only two fields in the summary will be non-zero:
+   `TotalFilesProcessed` and `TotalBytesProcessed`.
+
+   For snapshots rewritten using the `--exclude` options, the summary statistics
+   are updated accordingly.
+
+   https://github.com/restic/restic/issues/4942
+   https://github.com/restic/restic/pull/5185
+
+ * Enhancement #4948: Format exit errors as JSON when requested
+
+   Restic now formats error messages as JSON when the `--json` flag is used.
+
+   https://github.com/restic/restic/issues/4948
+   https://github.com/restic/restic/pull/4952
+
+ * Enhancement #4983: Add SLSA provenance to GHCR container images
+
+   Restic's GitHub Container Registry (GHCR) image build workflow now includes SLSA
+   (Supply-chain Levels for Software Artifacts) provenance generation.
+
+   Please see the restic documentation for more information about verifying SLSA
+   provenance.
+
+   https://github.com/restic/restic/issues/4983
+   https://github.com/restic/restic/pull/4999
+
+ * Enhancement #5054: Enable compression for ZIP archives in `dump` command
+
+   The `dump` command now compresses ZIP archives using the DEFLATE algorithm,
+   reducing the size of exported archives.
+
+   https://github.com/restic/restic/pull/5054
+
+ * Enhancement #5081: Add retry mechanism for loading repository config
+
+   Restic now retries loading the repository config file when opening a repository.
+   The `init` command now also retries backend operations.
+
+   https://github.com/restic/restic/issues/5081
+   https://github.com/restic/restic/pull/5095
+
+ * Enhancement #5089: Allow including/excluding extended file attributes during `restore`
+
+   The `restore` command now supports the `--exclude-xattr` and `--include-xattr`
+   options to control which extended file attributes will be restored. By default,
+   all attributes are restored.
+
+   https://github.com/restic/restic/issues/5089
+   https://github.com/restic/restic/pull/5129
+
+ * Enhancement #5092: Show count of deleted files and directories during `restore`
+
+   The `restore` command now reports the number of deleted files and directories,
+   both in the regular output and in the `files_deleted` field of the JSON output.
+
+   https://github.com/restic/restic/issues/5092
+   https://github.com/restic/restic/pull/5100
+
+ * Enhancement #5109: Make small pack size configurable for `prune`
+
+   The `prune` command now supports the `--repack-smaller-than` option that allows
+   repacking pack files smaller than a specified size.
+
+   https://github.com/restic/restic/issues/5109
+   https://github.com/restic/restic/pull/5183
+
+ * Enhancement #5119: Add start and end timestamps to `backup` JSON output
+
+   The JSON output of the `backup` command now includes `backup_start` and
+   `backup_end` timestamps, containing the start and end time of the backup.
+
+   https://github.com/restic/restic/pull/5119
+
+ * Enhancement #5131: Add DragonFlyBSD support
+
+   Restic can now be compiled on DragonflyBSD.
+
+   https://github.com/restic/restic/issues/5131
+   https://github.com/restic/restic/pull/5138
+
+ * Enhancement #5137: Make `tag` command print which snapshots were modified
+
+   The `tag` command now outputs which snapshots were modified along with their new
+   snapshot ID. The command supports the `--json` option for machine-readable
+   output.
+
+   https://github.com/restic/restic/issues/5137
+   https://github.com/restic/restic/pull/5144
+
+ * Enhancement #5141: Provide clear error message if AZURE_ACCOUNT_NAME is not set
+
+   If `AZURE_ACCOUNT_NAME` was not set, commands related to an Azure repository
+   would result in a misleading networking error. Restic now detect this and
+   provides a clear warning that the variable is not defined.
+
+   https://github.com/restic/restic/pull/5141
+
+ * Enhancement #5173: Add experimental S3 cold storage support
+
+   Introduce S3 backend options for transitioning pack files from cold to hot
+   storage on S3 and S3-compatible providers. Note: this only works for the
+   `prune`, `copy` and `restore` commands for now.
+
+   This experimental feature is gated behind the "s3-restore" feature flag.
+
+   https://github.com/restic/restic/issues/3202
+   https://github.com/restic/restic/issues/2504
+   https://github.com/restic/restic/pull/5173
+
+ * Enhancement #5174: Add xattr support for NetBSD 10+
+
+   Extended attribute support for `backup` and `restore` operations is now
+   available on NetBSD version 10 and later.
+
+   https://github.com/restic/restic/issues/5174
+   https://github.com/restic/restic/pull/5180
+
+ * Enhancement #5251: Improve retry handling for flaky `rclone` backends
+
+   Since restic 0.17.0, the backend retry mechanisms rely on backends correctly
+   reporting when a file does not exist. This is not always the case for some
+   `rclone` backends, which caused restic to stop retrying after the first failure.
+
+   For rclone, failed requests are now retried up to 5 times before giving up.
+
+   https://github.com/restic/restic/pull/5251
+
+ * Enhancement #5287: Make `recover` automatically rebuild index when needed
+
+   When trying to recover data from an interrupted snapshot, it was previously
+   necessary to manually run `repair index` before runnning `recover`. This now
+   happens automatically so that only `recover` is necessary.
+
+   https://github.com/restic/restic/issues/5287
+   https://github.com/restic/restic/pull/5296
+
+
 # Changelog for restic 0.17.3 (2024-11-08)
 The following sections list the changes in restic 0.17.3 relevant to
 restic users. The changes are ordered by importance.

CONTRIBUTING.md

@@ -202,6 +202,9 @@ we'll be glad to assist. Having a PR with failing integration tests is nothing
 to be ashamed of. In contrast, that happens regularly for all of us. That's
 what the tests are there for.
 
+More details of how to structure tests can be found here at
+[writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests).
+
 Git Commits
 -----------
 

VERSION

@@ -1 +1 @@
-0.17.3
+0.18.1-dev

build.go

@@ -36,7 +36,6 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 //go:build ignore_build_go
-// +build ignore_build_go
 
 package main
 
@@ -53,12 +52,14 @@ import (
 
 // config contains the configuration for the program to build.
 var config = Config{
-	Name:             "restic",                                 // name of the program executable and directory
+	Name:      "restic",                   // name of the program executable and directory
-	Namespace:        "github.com/restic/restic",               // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Namespace: "github.com/restic/restic", // subdir of GOPATH, e.g. "github.com/foo/bar"
-	Main:             "./cmd/restic",                           // package name for the main package
+	Main:      "./cmd/restic",             // package name for the main package
-	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
+	// disable_grpc_modules is necessary to reduce the binary size since cloud.google.com/go/storage v1.44.0
-	Tests:            []string{"./..."},                        // tests to run
+	// see https://github.com/googleapis/google-cloud-go/issues/11448
-	MinVersion:       GoVersion{Major: 1, Minor: 18, Patch: 0}, // minimum Go version supported
+	DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used
+	Tests:            []string{"./..."},                              // tests to run
+	MinVersion:       GoVersion{Major: 1, Minor: 24, Patch: 0},       // minimum Go version supported
 }
 
 // Config configures the build.
@@ -298,19 +299,21 @@ func (v GoVersion) AtLeast(other GoVersion) bool {
 		return true
 	}
 
+	if v.Major > other.Major {
+		return true
+	}
 	if v.Major < other.Major {
 		return false
 	}
 
+	if v.Minor > other.Minor {
+		return true
+	}
 	if v.Minor < other.Minor {
 		return false
 	}
 
-	if v.Patch < other.Patch {
+	return v.Patch >= other.Patch
-		return false
-	}
-
-	return true
 }
 
 func (v GoVersion) String() string {
@@ -380,12 +383,6 @@ func main() {
 		}
 	}
 
-	solarisMinVersion := GoVersion{Major: 1, Minor: 20, Patch: 0}
-	if env["GOARCH"] == "solaris" && !goVersion.AtLeast(solarisMinVersion) {
-		fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s for Solaris\n", goVersion, solarisMinVersion)
-		os.Exit(1)
-	}
-
 	verbosePrintf("detected Go version %v\n", goVersion)
 
 	preserveSymbols := false

changelog/0.18.0_2025-03-27/issue-1378

@@ -0,0 +1,7 @@
+Enhancement: Add JSON support to `check` command
+
+The `check` command now supports the `--json` option to output all statistics in 
+JSON format.
+
+https://github.com/restic/restic/issues/1378
+https://github.com/restic/restic/pull/5194

changelog/0.18.0_2025-03-27/issue-1843

@@ -0,0 +1,7 @@
+Bugfix: Correctly restore long filepaths' timestamp on old Windows
+
+The `restore` command now correctly restores timestamps for files with paths longer
+than 256 characters on Windows versions prior to Windows 10 1607.
+
+https://github.com/restic/restic/issues/1843
+https://github.com/restic/restic/pull/5061

changelog/0.18.0_2025-03-27/issue-2165

@@ -0,0 +1,13 @@
+Bugfix: Ignore disappeared backup source files
+
+The `backup` command now quietly skips files that are removed between directory
+listing and backup, instead of printing errors like:
+
+```
+error: lstat /some/file/name: no such file or directory
+```
+
+https://github.com/restic/restic/issues/2165
+https://github.com/restic/restic/issues/3098
+https://github.com/restic/restic/pull/5143
+https://github.com/restic/restic/pull/5145

changelog/0.18.0_2025-03-27/issue-2511

@@ -0,0 +1,7 @@
+Enhancement: Support generating shell completions to stdout
+
+The `generate` command now supports using `-` as the filename with the
+`--[shell]-completion` option to write the generated output to stdout.
+
+https://github.com/restic/restic/issues/2511
+https://github.com/restic/restic/pull/5053

changelog/0.18.0_2025-03-27/issue-3202

@@ -0,0 +1,11 @@
+Enhancement: Add experimental S3 cold storage support
+
+Introduce S3 backend options for transitioning pack files from cold to hot storage 
+on S3 and S3-compatible providers. Note: this only works for the `prune`, `copy` 
+and `restore` commands for now.
+
+This experimental feature is gated behind the "s3-restore" feature flag.
+
+https://github.com/restic/restic/pull/5173
+https://github.com/restic/restic/issues/3202
+https://github.com/restic/restic/issues/2504

changelog/0.18.0_2025-03-27/issue-3697

@@ -0,0 +1,12 @@
+Enhancement: Allow excluding online-only cloud files (e.g. OneDrive)
+
+Restic treated files synced using OneDrive Files On-Demand as though they
+were regular files. This caused issues with VSS and could cause OneDrive to
+download all files.
+
+Restic now allows the user to exclude these files when backing up with
+the `--exclude-cloud-files` option.
+
+https://github.com/restic/restic/issues/3697
+https://github.com/restic/restic/issues/4935
+https://github.com/restic/restic/pull/4990

changelog/0.18.0_2025-03-27/issue-4179

@@ -0,0 +1,8 @@
+Enhancement: Add `sort` option to `ls` command
+
+The `ls -l` command output can now be sorted using the new `--sort <field>`
+option for the fields `name`, `size`, `time` (same as `mtime`), `mtime`,
+`atime`, `ctime` and `extension`. A `--reverse` option is also available.
+
+https://github.com/restic/restic/issues/4179
+https://github.com/restic/restic/pull/5182

changelog/0.18.0_2025-03-27/issue-4433

@@ -0,0 +1,7 @@
+Enhancement: Change default sort order for `find` output
+
+The `find` command now sorts snapshots from newest to oldest by default. The
+previous oldest-to-newest order can be restored using the new `--reverse` option.
+
+https://github.com/restic/restic/issues/4433
+https://github.com/restic/restic/pull/5184

changelog/0.18.0_2025-03-27/issue-4521

@@ -0,0 +1,12 @@
+Enhancement: Add support for Microsoft Blob Storage access tiers
+
+The new `-o azure.access-tier=<tier>` option allows specifying the access tier
+(`Hot`, `Cool` or `Cold`) for objects created in Microsoft Blob Storage. If
+unspecified, the storage account's default tier is used.
+
+There is no official `Archive` storage support in restic, use this option at
+your own risk. To restore any data, it is necessary to manually warm up the
+required data in the `Archive` tier.
+
+https://github.com/restic/restic/issues/4521
+https://github.com/restic/restic/pull/5046

changelog/0.18.0_2025-03-27/issue-4942

@@ -0,0 +1,11 @@
+Enhancement: Add snapshot summary statistics to rewritten snapshots
+
+The `rewrite` command now supports a `--snapshot-summary` option to add
+statistics data to snapshots. Only two fields in the summary will be non-zero:
+`TotalFilesProcessed` and `TotalBytesProcessed`.
+
+For snapshots rewritten using the `--exclude` options, the summary
+statistics are updated accordingly.
+
+https://github.com/restic/restic/issues/4942
+https://github.com/restic/restic/pull/5185

changelog/0.18.0_2025-03-27/issue-4948

@@ -0,0 +1,6 @@
+Enhancement: Format exit errors as JSON when requested
+
+Restic now formats error messages as JSON when the `--json` flag is used.
+
+https://github.com/restic/restic/issues/4948
+https://github.com/restic/restic/pull/4952

changelog/0.18.0_2025-03-27/issue-4983

@@ -0,0 +1,10 @@
+Enhancement: Add SLSA provenance to GHCR container images
+
+Restic's GitHub Container Registry (GHCR) image build workflow now includes
+SLSA (Supply-chain Levels for Software Artifacts) provenance generation.
+
+Please see the restic documentation for more information about verifying SLSA
+provenance.
+
+https://github.com/restic/restic/issues/4983
+https://github.com/restic/restic/pull/4999

changelog/0.18.0_2025-03-27/issue-5081

@@ -0,0 +1,7 @@
+Enhancement: Add retry mechanism for loading repository config
+
+Restic now retries loading the repository config file when opening a repository.
+The `init` command now also retries backend operations.
+
+https://github.com/restic/restic/issues/5081
+https://github.com/restic/restic/pull/5095

changelog/0.18.0_2025-03-27/issue-5089

@@ -0,0 +1,8 @@
+Enhancement: Allow including/excluding extended file attributes during `restore`
+
+The `restore` command now supports the `--exclude-xattr` and `--include-xattr`
+options to control which extended file attributes will be restored. By default,
+all attributes are restored.
+
+https://github.com/restic/restic/issues/5089
+https://github.com/restic/restic/pull/5129

changelog/0.18.0_2025-03-27/issue-5092

@@ -0,0 +1,7 @@
+Enhancement: Show count of deleted files and directories during `restore`
+
+The `restore` command now reports the number of deleted files and directories,
+both in the regular output and in the `files_deleted` field of the JSON output.
+
+https://github.com/restic/restic/issues/5092
+https://github.com/restic/restic/pull/5100

changelog/0.18.0_2025-03-27/issue-5109

@@ -0,0 +1,7 @@
+Enhancement: Make small pack size configurable for `prune`
+
+The `prune` command now supports the `--repack-smaller-than` option that
+allows repacking pack files smaller than a specified size.
+
+https://github.com/restic/restic/issues/5109
+https://github.com/restic/restic/pull/5183

changelog/0.18.0_2025-03-27/issue-5131

@@ -0,0 +1,6 @@
+Enhancement: Add DragonFlyBSD support
+
+Restic can now be compiled on DragonflyBSD.
+
+https://github.com/restic/restic/issues/5131
+https://github.com/restic/restic/pull/5138

changelog/0.18.0_2025-03-27/issue-5137

@@ -0,0 +1,8 @@
+Enhancement: Make `tag` command print which snapshots were modified
+
+The `tag` command now outputs which snapshots were modified along with their
+new snapshot ID. The command supports the `--json` option for machine-readable
+output.
+
+https://github.com/restic/restic/issues/5137
+https://github.com/restic/restic/pull/5144

changelog/0.18.0_2025-03-27/issue-5174

@@ -0,0 +1,7 @@
+Enhancement: Add xattr support for NetBSD 10+
+
+Extended attribute support for `backup` and `restore` operations
+is now available on NetBSD version 10 and later.
+
+https://github.com/restic/restic/issues/5174
+https://github.com/restic/restic/pull/5180

changelog/0.18.0_2025-03-27/issue-5259

@@ -0,0 +1,16 @@
+Bugfix: Fix rare crash in command output
+
+Some commands could in rare cases crash when trying to print status messages
+and request retries at the same time, resulting in an error like the following:
+
+```
+panic: runtime error: slice bounds out of range [468:156]
+[...]
+github.com/restic/restic/internal/ui/termstatus.(*lineWriter).Write(...)
+	/restic/internal/ui/termstatus/stdio_wrapper.go:36 +0x136
+```
+
+This has now been fixed.
+
+https://github.com/restic/restic/issues/5259
+https://github.com/restic/restic/pull/5300

changelog/0.18.0_2025-03-27/issue-5287

@@ -0,0 +1,8 @@
+Enhancement: Make `recover` automatically rebuild index when needed
+
+When trying to recover data from an interrupted snapshot, it was previously
+necessary to manually run `repair index` before runnning `recover`. This now
+happens automatically so that only `recover` is necessary.
+
+https://github.com/restic/restic/issues/5287
+https://github.com/restic/restic/pull/5296

changelog/0.18.0_2025-03-27/issue-5291

@@ -0,0 +1,24 @@
+Security: Mitigate attack on content-defined chunking algorithm
+
+Restic uses [Rabin Fingerprints](https://restic.net/blog/2015-09-12/restic-foundation1-cdc/)
+for its content-defined chunker. The algorithm relies on a secret polynomial
+to split files into chunks.
+
+As shown in the paper "[Chunking Attacks on File Backup Services using Content-Defined Chunking](https://eprint.iacr.org/2025/532.pdf)"
+by Boris Alexeev, Colin Percival and Yan X Zhang, an
+attacker that can observe chunk sizes for a known file can derive the secret
+polynomial. Knowledge of the polynomial might in some cases allow an attacker
+to check whether certain large files are stored in a repository.
+
+A practical attack is nevertheless hard as restic merges multiple chunks into
+opaque pack files and by default processes multiple files in parallel. This
+likely prevents an attacker from matching pack files to the attacker-known file
+and thereby prevents the attack.
+
+Despite the low chances of a practical attack, restic now has added mitigation
+that randomizes how chunks are assembled into pack files. This prevents attackers
+from guessing which chunks are part of a pack file and thereby prevents learning
+the chunk sizes.
+
+https://github.com/restic/restic/issues/5291
+https://github.com/restic/restic/pull/5295

changelog/0.18.0_2025-03-27/pull-4938

@@ -0,0 +1,9 @@
+Change: Update dependencies and require Go 1.23 or newer
+
+We have updated all dependencies. Restic now requires Go 1.23 or newer to build.
+
+This also disables support for TLS versions older than TLS 1.2. On Windows,
+restic now requires at least Windows 10 or Windows Server 2016. On macOS,
+restic now requires at least macOS 11 Big Sur.
+
+https://github.com/restic/restic/pull/4938

changelog/0.18.0_2025-03-27/pull-5054

@@ -0,0 +1,6 @@
+Enhancement: Enable compression for ZIP archives in `dump` command
+
+The `dump` command now compresses ZIP archives using the DEFLATE algorithm,
+reducing the size of exported archives.
+
+https://github.com/restic/restic/pull/5054

changelog/0.18.0_2025-03-27/pull-5119

@@ -0,0 +1,6 @@
+Enhancement: Add start and end timestamps to `backup` JSON output
+
+The JSON output of the `backup` command now includes `backup_start` and
+`backup_end` timestamps, containing the start and end time of the backup.
+
+https://github.com/restic/restic/pull/5119

changelog/0.18.0_2025-03-27/pull-5141

@@ -0,0 +1,7 @@
+Enhancement: Provide clear error message if AZURE_ACCOUNT_NAME is not set
+
+If `AZURE_ACCOUNT_NAME` was not set, commands related to an Azure repository
+would result in a misleading networking error. Restic now detect this and
+provides a clear warning that the variable is not defined.
+
+https://github.com/restic/restic/pull/5141

changelog/0.18.0_2025-03-27/pull-5153

@@ -0,0 +1,7 @@
+Bugfix: Include root tree when searching using `find --tree`
+
+The `restic find --tree` command did not find trees referenced by
+`restic snapshot --json`. It now correctly includes the root tree
+when searching.
+
+https://github.com/restic/restic/pull/5153

changelog/0.18.0_2025-03-27/pull-5162

@@ -0,0 +1,8 @@
+Change: Graduate feature flags
+
+The `deprecate-legacy-index`, `deprecate-s3-legacy-layout`,
+`explicit-s3-anonymous-auth` and `safe-forget-keep-tags` features are
+now stable and can no longer be disabled. The corresponding feature flags
+will be removed in restic 0.19.0.
+
+https://github.com/restic/restic/pull/5162

changelog/0.18.0_2025-03-27/pull-5170

@@ -0,0 +1,22 @@
+Bugfix: Prevent Windows VSS event log 8194 warnings for backup with fs snapshot
+
+When running `backup` with the `--use-fs-snapshot` option in Windows with admin rights, event logs like
+
+```
+Volume Shadow Copy Service error: Unexpected error querying for the IVssWriterCallback interface.  hr = 0x80070005, Access is denied.
+. This is often caused by incorrect security settings in either the writer or requester process. 
+
+Operation:
+   Gathering Writer Data
+
+Context:
+   Writer Class Id: {e8132975-6f93-4464-a53e-1050253ae220}
+   Writer Name: System Writer
+   Writer Instance ID: {54b151ac-d27d-4628-9cb0-2bc40959f50f}
+```
+
+are created several times even though the backup itself succeeds. This has now been fixed.
+
+https://github.com/restic/restic/issues/5169
+https://github.com/restic/restic/pull/5170
+https://forum.restic.net/t/windows-shadow-copy-snapshot-vss-unexpected-provider-error/3674/2

changelog/0.18.0_2025-03-27/pull-5212

@@ -0,0 +1,8 @@
+Bugfix: Fix duplicate data handling in `prune --max-unused`
+
+The `prune --max-unused size` command did not correctly account for duplicate
+data. If a repository contained a large amount of duplicate data, this could
+previously result in pruning too little data. This has now been fixed.
+
+https://github.com/restic/restic/pull/5212
+https://forum.restic.net/t/restic-not-obeying-max-unused-parameter-on-prune/8879

changelog/0.18.0_2025-03-27/pull-5249

@@ -0,0 +1,10 @@
+Bugfix: Fix creation of oversized index by `repair index --read-all-packs`
+
+Since restic 0.17.0, the new index created by `repair index --read-all-packs` was
+written as a single large index. This significantly increased memory usage while
+loading the index.
+
+The index is now correctly split into multiple smaller indexes, and `repair index`
+now also automatically splits oversized indexes.
+
+https://github.com/restic/restic/pull/5249

changelog/0.18.0_2025-03-27/pull-5251

@@ -0,0 +1,9 @@
+Enhancement: Improve retry handling for flaky `rclone` backends
+
+Since restic 0.17.0, the backend retry mechanisms rely on backends correctly
+reporting when a file does not exist. This is not always the case for some
+`rclone` backends, which caused restic to stop retrying after the first failure.
+
+For rclone, failed requests are now retried up to 5 times before giving up.
+
+https://github.com/restic/restic/pull/5251

changelog/0.18.1_2025-09-21/issue-5324

@@ -0,0 +1,14 @@
+Bugfix: Correctly handle `backup --stdin-filename` with directory paths
+
+In restic 0.18.0, the `backup` command failed if a filename that includes
+at least a directory was passed to `--stdin-filename`. For example,
+`--stdin-filename /foo/bar` resulted in the following error:
+
+```
+Fatal: unable to save snapshot: open /foo: no such file or directory
+```
+
+This has now been fixed.
+
+https://github.com/restic/restic/issues/5324
+https://github.com/restic/restic/pull/5356

changelog/0.18.1_2025-09-21/issue-5325

@@ -0,0 +1,7 @@
+Bugfix: Accept `RESTIC_HOST` environment variable in `forget` command
+
+The `forget` command did not use the host name from the `RESTIC_HOST`
+environment variable when filtering snapshots. This has now been fixed.
+
+https://github.com/restic/restic/issues/5325
+https://github.com/restic/restic/pull/5327

changelog/0.18.1_2025-09-21/issue-5342

@@ -0,0 +1,7 @@
+Bugfix: Ignore "chmod not supported" errors when writing files
+
+Restic 0.18.0 introduced a bug that caused `chmod xxx: operation not supported`
+errors to appear when writing to a local file repository that did not support
+chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors.
+
+https://github.com/restic/restic/issues/5342

changelog/0.18.1_2025-09-21/issue-5344

@@ -0,0 +1,7 @@
+Bugfix: Ignore `EOPNOTSUPP` errors for extended attributes
+
+Restic 0.18.0 added extended attribute support for NetBSD 10+, but not all
+NetBSD filesystems support extended attributes. Other BSD systems can
+likewise return `EOPNOTSUPP`, so restic now ignores these errors.
+
+https://github.com/restic/restic/issues/5344

changelog/0.18.1_2025-09-21/issue-5429

@@ -0,0 +1,8 @@
+Bugfix: Stop retrying uploads when rest-server runs out of space
+
+When rest-server returns a `507 Insufficient Storage` error, it indicates
+that no more storage capacity is available. Restic now correctly stops
+retrying uploads in this case.
+
+https://github.com/restic/restic/issues/5429
+https://github.com/restic/restic/pull/5452

changelog/0.18.1_2025-09-21/issue-5467

@@ -0,0 +1,27 @@
+Bugfix: Improve handling of download retries in `check` command
+
+In very rare cases, the `check` command could unnecessarily report repository
+damage if the backend returned incomplete, corrupted data on the first download
+try which is afterwards resolved by a download retry.
+
+This could result in an error output like the following:
+
+```
+Load(<data/34567890ab>, 33918928, 0) returned error, retrying after 871.35598ms: readFull: unexpected EOF
+Load(<data/34567890ab>, 33918928, 0) operation successful after 1 retries
+check successful on second attempt, original error pack 34567890ab[...] contains 6 errors: [blob 12345678[...]: decrypting blob <data/12345678> from 34567890 failed: ciphertext verification failed ...]
+[...]
+Fatal: repository contains errors
+```
+
+This fix only applies to a very specific case where the log shows
+`operation successful after 1 retries` followed by a
+`check successful on second attempt, original error` that only reports
+`ciphertext verification failed` errors in the pack file. If any other errors
+are reported in the pack file, then the repository still has to be considered
+as damaged.
+
+Now, only the check result of the last download retry is reported as intended.
+
+https://github.com/restic/restic/issues/5467
+https://github.com/restic/restic/pull/5495

changelog/0.18.1_2025-09-21/pull-5421

@@ -0,0 +1,7 @@
+Bugfix: Fix rare crash if directory is removed during backup
+
+In restic 0.18.0, the `backup` command could crash if a directory was removed
+between reading its metadata and listing its directory content. This has now
+been fixed.
+
+https://github.com/restic/restic/pull/5421

changelog/TEMPLATE

@@ -1,12 +1,20 @@
 # The first line must start with Bugfix:, Enhancement: or Change:,
-# including the colon. Use present tense and the imperative mood. Remove
+# including the colon. 'Change:' is for breaking changes only.
-# lines starting with '#' from this template.
+# Documentation-only changes do not get a changelog entry.
-Enhancement: Allow custom bar in the foo command
+# Include the affected command in the summary if relevant.
+#
+# Use present tense and the imperative mood. Remove lines starting
+# with '#' from this template.
+Enhancement: Allow custom bar in the `foo` command
 
 # Describe the problem in the past tense, the new behavior in the present
 # tense. Mention the affected commands, backends, operating systems, etc.
-# Focus on user-facing behavior, not the implementation.
+# If the problem description just says that a feature was missing, then
+# only explain the new behavior. Aim for a short and concise description.
 # Use "Restic now ..." instead of "We have changed ...".
+#
+# Focus on user-facing behavior, not the implementation. The description should
+# be understandable for a regular user without knowledge of the implementation.
 
 Restic foo always used the system-wide bar when deciding how to frob an
 item in the `baz` backend. It now permits selecting the bar with `--bar`

changelog/unreleased/issue-3326

@@ -0,0 +1,9 @@
+Enhancement: `restic check` for specified snapshot(s) via snapshot filtering
+
+Snapshots can now be specified for the command `restic check` on the command line
+via the standard snapshot filter, (`--tag`, `--host`, `--path` or specifying
+snapshot IDs directly) and will be used for checking the packfiles used by these snapshots.
+
+https://github.com/restic/restic/issues/3326
+https://github.com/restic/restic/pull/5469
+https://github.com/restic/restic/pull/5644

changelog/unreleased/issue-3572

@@ -0,0 +1,9 @@
+Enhancement: Support restoring ownership by name on UNIX systems
+
+Restic restore used to restore file ownership on UNIX systems by UID and GID.
+It now allows restoring the file ownership by user name and group name with `--ownership-by-name`.
+This allows restoring snapshots on a system where the UID/GID are not the same as they were on the system where the snapshot was created.
+However it does not include support for POSIX ACLs, which are still restored by their numeric value.
+
+https://github.com/restic/restic/issues/3572
+https://github.com/restic/restic/pull/5449

changelog/unreleased/issue-3738

@@ -0,0 +1,8 @@
+Enhancement: Allow Github personal access token to be specified for `self-update`
+
+`restic self-update` previously only used unauthenticated GitHub API requests when checking for the latest release. This caused some users sharing IP addresses to hit the GitHub rate limit, resulting in a 403 Forbidden error and preventing updates.
+
+Restic still uses unauthenticated requests by default, but it now optionally supports authenticated GitHub API requests during `self-update`. Users can set the `$GITHUB_ACCESS_TOKEN` environment variable to use a [personal access token](https://github.com/settings/tokens) for this effect, avoiding update failures due to rate limiting.
+
+https://github.com/restic/restic/issues/3738
+https://github.com/restic/restic/pull/5568

changelog/unreleased/issue-4278

@@ -0,0 +1,12 @@
+Enhancement: Support include filters in `rewrite` command
+
+The enhancement enables the standard include filter options
+      --iinclude pattern     same as --include pattern but ignores the casing of filenames
+      --iinclude-file file   same as --include-file but ignores casing of filenames in patterns
+  -i, --include pattern      include a pattern (can be specified multiple times)
+      --include-file file    read include patterns from a file (can be specified multiple times)
+
+The exclusion or inclusion of filter parameters is exclusive, as in other commands.
+
+https://github.com/restic/restic/issues/4278
+https://github.com/restic/restic/pull/5191

changelog/unreleased/issue-4467

@@ -0,0 +1,11 @@
+Bugfix: Exit with code 3 when some `backup` source files do not exist
+
+Restic used to exit with code 0 even when some backup sources did not exist. Restic
+would exit with code 3 only when child directories or files did not exist. This
+could cause confusion and unexpected behavior in scripts that relied on the exit
+code to determine if the backup was successful.
+
+Restic now exits with code 3 when some backup sources do not exist.
+
+https://github.com/restic/restic/issues/4467
+https://github.com/restic/restic/pull/5347

changelog/unreleased/issue-4728

@@ -0,0 +1,7 @@
+Enhancement: Added support for zstd compression levels `fastest` and `better`
+
+Restic now supports the zstd compression modes `fastest` and `better`. Set the
+environment variable `RESTIC_COMPRESSION` to `fastest` or `better` to use these
+compression levels. This can also be set with the `--compression` flag.
+
+https://github.com/restic/restic/issues/4728

changelog/unreleased/issue-4868

@@ -0,0 +1,14 @@
+Enhancement: Include repository id in filesystem name used by `mount`
+
+The filesystem created by restic's `mount` command now includes the repository
+id in the filesystem name. The repository id is printed by restic when opening
+a repository or can be looked up using `restic cat config`.
+
+```
+[restic-user@hostname restic]$ df ./test-mount/
+Filesystem        1K-blocks  Used Available Use% Mounted on
+restic:d3b07384d1         0     0         0    - /mnt/my-restic-repo
+```
+
+https://github.com/restic/restic/issues/4868
+https://github.com/restic/restic/pull/5243

changelog/unreleased/issue-5233

@@ -0,0 +1,8 @@
+Bugfix: forget command returns exit code 3 on partial removal of snapshots
+
+The `forget` command now returns exit code 3 when it fails to remove one or
+more snapshots. Previously, it returned exit code 0, which could lead to
+confusion if the command was used in a script.
+
+https://github.com/restic/restic/issues/5233
+https://github.com/restic/restic/pull/5322

changelog/unreleased/issue-5258

@@ -0,0 +1,7 @@
+Bugfix: Exit with correct code on SIGINT
+
+Restic previously returned exit code 1 on SIGINT, which is incorrect. 
+Restic now returns 130 on SIGINT.
+
+https://github.com/restic/restic/issues/5258
+https://github.com/restic/restic/pull/5363

changelog/unreleased/issue-5280

@@ -0,0 +1,7 @@
+Bugfix: `restic find` now checks for correct ordering of time related options
+
+`restic find` now immediately fails with an error if both `--oldest` and `--newest` are specified
+and `--oldest` is a timestamp after `--newest`.
+
+https://github.com/restic/restic/issues/5280
+https://github.com/restic/restic/pull/5310

changelog/unreleased/issue-5352

@@ -0,0 +1,11 @@
+Enhancement: Add support for --exclude-cloud-files on macOS (e.g. iCloud drive)
+
+Restic treated files stored in iCloud drive as though they were regular files.
+This caused restic to download all files (including files marked as cloud only) while iterating over them.
+
+Restic now allows the user to exclude these files when backing up with the `--exclude-cloud-files` option.
+
+Works from Sonoma (macOS 14.0) onwards. Older macOS versions materialize files when `stat` is called on the file.
+
+https://github.com/restic/restic/pull/4990
+https://github.com/restic/restic/issues/5352

changelog/unreleased/issue-5354

@@ -0,0 +1,14 @@
+Bugfix: Allow use of rclone/sftp backend when running restic in background
+
+When starting restic in the background, this could result in unexpected behavior
+when using the rclone or sftp backend.
+
+For example running `restic -r rclone:./example --insecure-no-password init &`
+could cause the calling `bash` shell to exit unexpectedly.
+
+This has been fixed.
+
+https://github.com/restic/restic/issues/5354
+https://github.com/restic/restic/pull/5358
+https://github.com/restic/restic/pull/5493
+https://github.com/restic/restic/pull/5494

changelog/unreleased/issue-5383

@@ -0,0 +1,10 @@
+Enhancement: Reduce progress bar refresh rates to reduce energy usage
+
+Progress bars were updated with 60fps which can cause high CPU or GPU usage
+for some terminal emulators. Reduce it to 10fps to conserve energy.
+In addition, this lower frequency seem to be necessary to allow selecting
+anything in the terminal with certain terminal emulators.
+
+https://github.com/restic/restic/issues/5383
+https://github.com/restic/restic/pull/5551
+https://github.com/restic/restic/pull/5626

changelog/unreleased/issue-5440

@@ -0,0 +1,12 @@
+Enhancement: Allow overriding RESTIC_HOST environment variable with --host flag
+
+When the `RESTIC_HOST` environment variable was set, there was no way to list or
+operate on snapshots from all hosts, as the environment variable would always
+filter to that specific host. Restic now allows overriding `RESTIC_HOST` by
+explicitly providing the `--host` flag with an empty string (e.g., `--host=""` or
+`--host=`), which will show snapshots from all hosts. This works for all commands
+that support snapshot filtering: `snapshots`, `forget`, `find`, `stats`, `copy`,
+`tag`, `repair snapshots`, `rewrite`, `mount`, `restore`, `dump`, and `ls`.
+
+https://github.com/restic/restic/issues/5440
+https://github.com/restic/restic/pull/5541

changelog/unreleased/issue-5453

@@ -0,0 +1,10 @@
+Enhancement: `copy` copies snapshots in batches
+
+The `copy` command used to copy snapshots individually, even if this resulted in creating pack files
+smaller than the target pack size. In particular, this resulted in many small files
+when copying small incremental snapshots.
+
+Now, `copy` copies multiple snapshots at once to avoid creating small files.
+
+https://github.com/restic/restic/issues/5175
+https://github.com/restic/restic/pull/5464

changelog/unreleased/issue-5477

@@ -0,0 +1,7 @@
+Bugfix: Password prompt was sometimes not shown
+
+The password prompt for a repository was sometimes not shown when running
+the `backup -v` command. This has been fixed.
+
+https://github.com/restic/restic/issues/5477
+https://github.com/restic/restic/pull/5554

changelog/unreleased/issue-5487

@@ -0,0 +1,8 @@
+Bugfix: Mark files as readonly when using the SFTP backend
+
+Files created by the SFTP backend previously allowed writes to those files.
+Restic now restricts the file permissions on SFTP backend to readonly.
+This change only has an effect for sftp servers with support for the chmod operation.
+
+https://github.com/restic/restic/issues/5487
+https://github.com/restic/restic/pull/5497

changelog/unreleased/issue-5531

@@ -0,0 +1,15 @@
+Enhancement: Reduce Azure storage costs by optimizing upload method
+
+Restic previously used Azure's PutBlock and PutBlockList APIs for all file
+uploads, which resulted in two transactions per file and doubled the storage
+operation costs. For backups with many pack files, this could lead to
+significant Azure storage transaction fees.
+
+Restic now uses the more efficient PutBlob API for files up to 256 MiB,
+requiring only a single transaction per file. This reduces Azure storage
+operation costs by approximately 50% for typical backup workloads. Files
+larger than 256 MiB continue to use the block-based upload method as required
+by Azure's API limits.
+
+https://github.com/restic/restic/issues/5531
+https://github.com/restic/restic/pull/5544

changelog/unreleased/issue-5586

@@ -0,0 +1,7 @@
+Bugfix: correctly handle `snapshots --group-by` in combination with `--latest`
+
+For the `snapshots` command, the `--latest` option did not correctly handle the
+case where an non-default value was passed to `--group-by`. This has been fixed.
+
+https://github.com/restic/restic/issues/5586
+https://github.com/restic/restic/pull/5601

changelog/unreleased/issue-5595

@@ -0,0 +1,8 @@
+Bugfix: Fix "chmod not supported" errors when unlocking
+
+Restic 0.18.0 introduced a bug that caused "chmod xxx: operation not supported"
+errors to appear when unlocking with a stale lock, on a local file repository
+that did not support chmod (like CIFS or WebDAV mounted via FUSE). Restic now
+just doesn't bother calling chmod in that case on Unix, as it is unnecessary.
+
+https://github.com/restic/restic/issues/5595

changelog/unreleased/pull-4938

@@ -0,0 +1,5 @@
+Change: Update dependencies and require Go 1.24 or newer
+
+We have updated all dependencies. Restic now requires Go 1.24 or newer to build.
+
+https://github.com/restic/restic/pull/5619

changelog/unreleased/pull-5319

@@ -0,0 +1,9 @@
+Enhancement: add more status counters to `restic copy`
+
+`restic copy` now produces more status counters in text format. The new counters
+are the number of blobs to copy, their size on disk and the number of packfiles
+used from the source repository. The additional statistics is only produced when
+the `--verbose` option is specified.
+
+https://github.com/restic/restic/issues/5175
+https://github.com/restic/restic/pull/5319

changelog/unreleased/pull-5424

@@ -0,0 +1,11 @@
+Enhancement: Enable file system privileges on Windows before access
+
+Restic attempted to enable Windows file system privileges when
+reading or writing security descriptors - after potentially being wholly
+denied access to previous items. It also read file extended attributes without
+using the privilege, possibly missing them and producing errors.
+
+Restic now attempts to enable all file system privileges before any file
+access. It also requests extended attribute reads use the backup privilege.
+
+https://github.com/restic/restic/pull/5424

changelog/unreleased/pull-5448

@@ -0,0 +1,11 @@
+Enhancement: Allow nice and ionice configuration for restic containers
+
+The official restic docker now supports the following environment variables:
+
+`NICE`: set the desired nice scheduling.  See `man nice`.
+`IONICE_CLASS`: set the desired I/O scheduling class.  See `man ionice`.  Note that real time support requires the invoker to manually add the `SYS_NICE` capability.
+`IONICE_PRIORITY`: set the prioritization for ionice in the given `IONICE_CLASS`.  This does nothing without `IONICE_CLASS`, but defaults to `4` (no priority, no penalties).
+
+See https://restic.readthedocs.io/en/stable/020_installation.html#docker-container for further details.
+
+https://github.com/restic/restic/pull/5448

changelog/unreleased/pull-5465

@@ -0,0 +1,10 @@
+Bugfix: Correctly restore ACL inheritance state on Windows
+
+Since the introduction of Security Descriptor backups in restic 0.17.0, the inheritance property of Access Control Entries (ACEs) was not restored correctly. This resulted in all restored permissions being marked as explicit (IsInherited: False), even if they were originally inherited from a parent folder.
+
+The issue was caused by sending conflicting inheritance flags (PROTECTED_... and UNPROTECTED_...) to the Windows API during the restore process. The API would default to the more restrictive PROTECTED state, effectively disabling inheritance.
+
+This has been fixed by ensuring that only the correct, non-conflicting inheritance flag is used when applying the security descriptor, preserving the original permission structure from the backup.
+
+https://github.com/restic/restic/pull/5465
+https://github.com/restic/restic/issues/5427

changelog/unreleased/pull-5523

@@ -0,0 +1,6 @@
+Enhancement: Add OpenContainers labels to Dockerfile.release
+
+The restic Docker image now includes labels from the OpenContainers Annotations Spec.
+This information can be used by third party services.
+
+https://github.com/restic/restic/pull/5523

changelog/unreleased/pull-5588

@@ -0,0 +1,10 @@
+Enhancement: Display timezone information in snapshots output
+
+The `snapshots` command now displays which timezone is being used to show
+timestamps. Since snapshots can be created in different timezones but are
+always displayed in the local timezone, a footer line is now shown indicating
+the timezone used for display (e.g., "Timestamps shown in CET timezone").
+This helps prevent confusion when comparing snapshots in a multi-user
+environment.
+
+https://github.com/restic/restic/pull/5588

changelog/unreleased/pull-5592

@@ -0,0 +1,7 @@
+Bugfix: Return error if `RESTIC_PACK_SIZE` contains invalid value
+
+If the environment variable `RESTIC_PACK_SIZE` could not be parsed, then
+restic ignored its value. Now, the restic commands fail with an error, unless
+the command-line option `--pack-size` was specified.
+
+https://github.com/restic/restic/pull/5592

changelog/unreleased/pull-5610

@@ -0,0 +1,7 @@
+Enhancement: reduce memory usage of check/copy/diff/stats commands
+
+We have optimized the memory usage of the `check`, `copy`, `diff` and
+`stats` commands. These now require less memory when processing large
+snapshots.
+
+https://github.com/restic/restic/pull/5610

changelog/unreleased/pull-5664

@@ -0,0 +1,6 @@
+Bugfix: restic find --pack <tree-pack> did not produce output for tree packs
+
+`restic find --pack` now produces output for a tree related packfile.
+
+https://github.com/restic/restic/issues/5280
+https://github.com/restic/restic/pull/5664

changelog/unreleased/pull-5718

@@ -0,0 +1,9 @@
+Enhancement: stricter early mountpoint validation in `mount`
+
+`restic mount` accepted parameters that would lead to a FUSE mount operation
+failing after having done computationally intensive work to prepare the mount.
+The `mountpoint` argument supplied must now refer to the name of a directory
+that the current user can access and write to, otherwise `restic mount` will
+exit with an error before interacting with the repository.
+
+https://github.com/restic/restic/pull/5718

cmd/restic/cleanup.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"fmt"
+	"io"
 	"os"
 	"os/signal"
 	"syscall"
@@ -9,26 +11,27 @@ import (
 	"github.com/restic/restic/internal/debug"
 )
 
-func createGlobalContext() context.Context {
+func createGlobalContext(stderr io.Writer) context.Context {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	ch := make(chan os.Signal, 1)
-	go cleanupHandler(ch, cancel)
+	go cleanupHandler(ch, cancel, stderr)
 	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
 
 	return ctx
 }
 
 // cleanupHandler handles the SIGINT and SIGTERM signals.
-func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
+func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc, stderr io.Writer) {
 	s := <-c
 	debug.Log("signal %v received, cleaning up", s)
-	Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
+	// ignore error as there's no good way to handle it
+	_, _ = fmt.Fprintf(stderr, "\rsignal %v received, cleaning up \n", s)
 
 	if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
-		_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
+		_, _ = stderr.Write([]byte("\n--- STACKTRACE START ---\n\n"))
-		_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
+		_, _ = stderr.Write([]byte(debug.DumpStacktrace()))
-		_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
+		_, _ = stderr.Write([]byte("\n--- STACKTRACE END ---\n"))
 	}
 
 	cancel()

cmd/restic/cmd_backup.go

@@ -15,23 +15,30 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/archiver"
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/backup"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-var cmdBackup = &cobra.Command{
+func newBackupCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "backup [flags] [FILE/DIR] ...",
+	var opts BackupOptions
-	Short: "Create a new backup of files and/or directories",
+
-	Long: `
+	cmd := &cobra.Command{
+		Use:   "backup [flags] [FILE/DIR] ...",
+		Short: "Create a new backup of files and/or directories",
+		Long: `
 The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 
@@ -45,40 +52,43 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	PreRun: func(_ *cobra.Command, _ []string) {
+		PreRun: func(_ *cobra.Command, _ []string) {
-		if backupOptions.Host == "" {
+			if opts.Host == "" {
-			hostname, err := os.Hostname()
+				hostname, err := os.Hostname()
-			if err != nil {
+				if err != nil {
-				debug.Log("os.Hostname() returned err: %v", err)
+					debug.Log("os.Hostname() returned err: %v", err)
-				return
+					return
+				}
+				opts.Host = hostname
 			}
-			backupOptions.Host = hostname
+		},
-		}
+		GroupID:           cmdGroupDefault,
-	},
+		DisableAutoGenTag: true,
-	GroupID:           cmdGroupDefault,
+		RunE: func(cmd *cobra.Command, args []string) error {
-	DisableAutoGenTag: true,
+			return runBackup(cmd.Context(), opts, *globalOptions, globalOptions.Term, args)
-	RunE: func(cmd *cobra.Command, args []string) error {
+		},
-		term, cancel := setupTermstatus()
+	}
-		defer cancel()
+
-		return runBackup(cmd.Context(), backupOptions, globalOptions, term, args)
+	opts.AddFlags(cmd.Flags())
-	},
+	return cmd
 }
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	excludePatternOptions
+	filter.ExcludePatternOptions
 
 	Parent            string
-	GroupBy           restic.SnapshotGroupByOptions
+	GroupBy           data.SnapshotGroupByOptions
 	Force             bool
 	ExcludeOtherFS    bool
 	ExcludeIfPresent  []string
 	ExcludeCaches     bool
 	ExcludeLargerThan string
+	ExcludeCloudFiles bool
 	Stdin             bool
 	StdinFilename     string
 	StdinCommand      bool
-	Tags              restic.TagLists
+	Tags              data.TagLists
 	Host              string
 	FilesFrom         []string
 	FilesFromVerbatim []string
@@ -94,70 +104,72 @@ type BackupOptions struct {
 	SkipIfUnchanged   bool
 }
 
-var backupOptions BackupOptions
+func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) {
-var backupFSTestHook func(fs fs.FS) fs.FS
+	f.StringVar(&opts.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
+	opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true}
+	f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
+	f.BoolVarP(&opts.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
 
-// ErrInvalidSourceData is used to report an incomplete backup
+	opts.ExcludePatternOptions.Add(f)
-var ErrInvalidSourceData = errors.New("at least one source file could not be read")
 
-func init() {
+	f.BoolVarP(&opts.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
-	cmdRoot.AddCommand(cmdBackup)
+	f.StringArrayVar(&opts.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
-
+	f.BoolVar(&opts.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
-	f := cmdBackup.Flags()
+	f.StringVar(&opts.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
+	f.BoolVar(&opts.Stdin, "stdin", false, "read backup from stdin")
-	backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+	f.StringVar(&opts.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
-	f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
+	f.BoolVar(&opts.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
-	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
+	f.Var(&opts.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
-
+	f.UintVar(&opts.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
-	initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
+	f.StringVarP(&opts.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
-
+	f.StringVar(&opts.Host, "hostname", "", "set the `hostname` for the snapshot manually")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
-	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
-	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
-	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
-	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
-	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
-	f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
-	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
-	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
-	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
-	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
 		// MarkDeprecated only returns an error when the flag could not be found
 		panic(err)
 	}
-	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&opts.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&opts.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
-	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&opts.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
-	f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
+	f.StringVar(&opts.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
-	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
+	f.BoolVar(&opts.WithAtime, "with-atime", false, "store the atime for all files and directories")
-	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files")
+	f.BoolVar(&opts.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files")
-	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	f.BoolVar(&opts.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
-	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
+	f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
-	f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
+	f.BoolVar(&opts.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
 	if runtime.GOOS == "windows" {
-		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
+		f.BoolVar(&opts.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
-	f.BoolVar(&backupOptions.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
+	if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+		f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive, iCloud drive, …)")
+	}
+	f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
 
 	// parse read concurrency from env, on error the default value will be used
 	readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
-	backupOptions.ReadConcurrency = uint(readConcurrency)
+	opts.ReadConcurrency = uint(readConcurrency)
 
 	// parse host from env, if not exists or empty the default value will be used
 	if host := os.Getenv("RESTIC_HOST"); host != "" {
-		backupOptions.Host = host
+		opts.Host = host
 	}
 }
 
+var backupFSTestHook func(fs fs.FS) fs.FS
+
+// ErrInvalidSourceData is used to report an incomplete backup
+var ErrInvalidSourceData = errors.New("at least one source file could not be read")
+
+// ErrNoSourceData is used to report that no source data was found
+var ErrNoSourceData = errors.Fatal("all source directories/files do not exist")
+
 // filterExisting returns a slice of all existing items, or an error if no
 // items exist at all.
-func filterExisting(items []string) (result []string, err error) {
+func filterExisting(items []string, warnf func(msg string, args ...interface{})) (result []string, err error) {
 	for _, item := range items {
 		_, err := fs.Lstat(item)
 		if errors.Is(err, os.ErrNotExist) {
-			Warnf("%v does not exist, skipping\n", item)
+			warnf("%v does not exist, skipping\n", item)
 			continue
 		}
 
@@ -165,10 +177,12 @@ func filterExisting(items []string) (result []string, err error) {
 	}
 
 	if len(result) == 0 {
-		return nil, errors.Fatal("all source directories/files do not exist")
+		return nil, ErrNoSourceData
+	} else if len(result) < len(items) {
+		return result, ErrInvalidSourceData
 	}
 
-	return
+	return result, nil
 }
 
 // readLines reads all lines from the named file and returns them as a
@@ -177,7 +191,7 @@ func filterExisting(items []string) (result []string, err error) {
 // If filename is empty, readPatternsFromFile returns an empty slice.
 // If filename is a dash (-), readPatternsFromFile will read the lines from the
 // standard input.
-func readLines(filename string) ([]string, error) {
+func readLines(filename string, stdin io.ReadCloser) ([]string, error) {
 	if filename == "" {
 		return nil, nil
 	}
@@ -188,7 +202,7 @@ func readLines(filename string) ([]string, error) {
 	)
 
 	if filename == "-" {
-		data, err = io.ReadAll(os.Stdin)
+		data, err = io.ReadAll(stdin)
 	} else {
 		data, err = textfile.Read(filename)
 	}
@@ -213,8 +227,8 @@ func readLines(filename string) ([]string, error) {
 // readFilenamesFromFileRaw reads a list of filenames from the given file,
 // or stdin if filename is "-". Each filename is terminated by a zero byte,
 // which is stripped off.
-func readFilenamesFromFileRaw(filename string) (names []string, err error) {
+func readFilenamesFromFileRaw(filename string, stdin io.ReadCloser) (names []string, err error) {
-	f := os.Stdin
+	f := stdin
 	if filename != "-" {
 		if f, err = os.Open(filename); err != nil {
 			return nil, err
@@ -263,8 +277,8 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
 }
 
 // Check returns an error when an invalid combination of options was set.
-func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
+func (opts BackupOptions) Check(gopts global.Options, args []string) error {
-	if gopts.password == "" && !gopts.InsecureNoPassword {
+	if gopts.Password == "" && !gopts.InsecureNoPassword {
 		if opts.Stdin {
 			return errors.Fatal("cannot read both password and data from stdin")
 		}
@@ -298,9 +312,9 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 
 // collectRejectByNameFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path only
-func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, warnf func(msg string, args ...interface{})) (fs []archiver.RejectByNameFunc, err error) {
 	// exclude restic cache
-	if repo.Cache != nil {
+	if repo.Cache() != nil {
 		f, err := rejectResticCache(repo)
 		if err != nil {
 			return nil, err
@@ -309,23 +323,12 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
 		fs = append(fs, f)
 	}
 
-	fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
+	fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(warnf)
 	if err != nil {
 		return nil, err
 	}
-	fs = append(fs, fsPatterns...)
+	for _, pat := range fsPatterns {
-
+		fs = append(fs, archiver.RejectByNameFunc(pat))
-	if opts.ExcludeCaches {
-		opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
-	}
-
-	for _, spec := range opts.ExcludeIfPresent {
-		f, err := rejectIfPresent(spec)
-		if err != nil {
-			return nil, err
-		}
-
-		fs = append(fs, f)
 	}
 
 	return fs, nil
@@ -333,35 +336,61 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
 
 // collectRejectFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path and file info
-func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, err error) {
+func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf func(msg string, args ...interface{})) (funcs []archiver.RejectFunc, err error) {
 	// allowed devices
-	if opts.ExcludeOtherFS && !opts.Stdin {
+	if opts.ExcludeOtherFS && !opts.Stdin && !opts.StdinCommand {
-		f, err := rejectByDevice(targets)
+		f, err := archiver.RejectByDevice(targets, fs)
 		if err != nil {
 			return nil, err
 		}
-		fs = append(fs, f)
+		funcs = append(funcs, f)
 	}
 
-	if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin {
+	if len(opts.ExcludeLargerThan) != 0 && !opts.Stdin && !opts.StdinCommand {
-		f, err := rejectBySize(opts.ExcludeLargerThan)
+		maxSize, err := ui.ParseBytes(opts.ExcludeLargerThan)
 		if err != nil {
 			return nil, err
 		}
-		fs = append(fs, f)
+
+		f, err := archiver.RejectBySize(maxSize)
+		if err != nil {
+			return nil, err
+		}
+		funcs = append(funcs, f)
 	}
 
-	return fs, nil
+	if opts.ExcludeCloudFiles && !opts.Stdin && !opts.StdinCommand {
+		f, err := archiver.RejectCloudFiles(warnf)
+		if err != nil {
+			return nil, err
+		}
+		funcs = append(funcs, f)
+	}
+
+	if opts.ExcludeCaches {
+		opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
+	}
+
+	for _, spec := range opts.ExcludeIfPresent {
+		f, err := archiver.RejectIfPresent(spec, warnf)
+		if err != nil {
+			return nil, err
+		}
+
+		funcs = append(funcs, f)
+	}
+
+	return funcs, nil
 }
 
 // collectTargets returns a list of target files/dirs from several sources.
-func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
+func collectTargets(opts BackupOptions, args []string, warnf func(msg string, args ...interface{}), stdin io.ReadCloser) (targets []string, err error) {
 	if opts.Stdin || opts.StdinCommand {
 		return nil, nil
 	}
 
 	for _, file := range opts.FilesFrom {
-		fromfile, err := readLines(file)
+		fromfile, err := readLines(file, stdin)
 		if err != nil {
 			return nil, err
 		}
@@ -379,14 +408,14 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 				return nil, fmt.Errorf("pattern: %s: %w", line, err)
 			}
 			if len(expanded) == 0 {
-				Warnf("pattern %q does not match any files, skipping\n", line)
+				warnf("pattern %q does not match any files, skipping\n", line)
 			}
 			targets = append(targets, expanded...)
 		}
 	}
 
 	for _, file := range opts.FilesFromVerbatim {
-		fromfile, err := readLines(file)
+		fromfile, err := readLines(file, stdin)
 		if err != nil {
 			return nil, err
 		}
@@ -399,7 +428,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 	}
 
 	for _, file := range opts.FilesFromRaw {
-		fromfile, err := readFilenamesFromFileRaw(file)
+		fromfile, err := readFilenamesFromFileRaw(file, stdin)
 		if err != nil {
 			return nil, err
 		}
@@ -413,17 +442,12 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 		return nil, errors.Fatal("nothing to backup, please specify source files/dirs")
 	}
 
-	targets, err = filterExisting(targets)
+	return filterExisting(targets, warnf)
-	if err != nil {
-		return nil, err
-	}
-
-	return targets, nil
 }
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
+func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*data.Snapshot, error) {
 	if opts.Force {
 		return nil, nil
 	}
@@ -432,7 +456,7 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o
 	if snName == "" {
 		snName = "latest"
 	}
-	f := restic.SnapshotFilter{TimestampLimit: timeStampLimit}
+	f := data.SnapshotFilter{TimestampLimit: timeStampLimit}
 	if opts.GroupBy.Host {
 		f.Hosts = []string{opts.Host}
 	}
@@ -440,23 +464,29 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o
 		f.Paths = targets
 	}
 	if opts.GroupBy.Tag {
-		f.Tags = []restic.TagList{opts.Tags.Flatten()}
+		f.Tags = []data.TagList{opts.Tags.Flatten()}
 	}
 
 	sn, _, err := f.FindLatest(ctx, repo, repo, snName)
 	// Snapshot not found is ok if no explicit parent was set
-	if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
+	if opts.Parent == "" && errors.Is(err, data.ErrNoSnapshotFound) {
 		err = nil
 	}
 	return sn, err
 }
 
-func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
+func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, term ui.Terminal, args []string) error {
 	var vsscfg fs.VSSConfig
 	var err error
 
+	var printer backup.ProgressPrinter
+	if gopts.JSON {
+		printer = backup.NewJSONProgress(term, gopts.Verbosity)
+	} else {
+		printer = backup.NewTextProgress(term, gopts.Verbosity)
+	}
 	if runtime.GOOS == "windows" {
-		if vsscfg, err = fs.ParseVSSConfig(gopts.extended); err != nil {
+		if vsscfg, err = fs.ParseVSSConfig(gopts.Extended); err != nil {
 			return err
 		}
 	}
@@ -466,53 +496,46 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		return err
 	}
 
-	targets, err := collectTargets(opts, args)
+	success := true
+	targets, err := collectTargets(opts, args, printer.E, term.InputRaw())
 	if err != nil {
-		return err
+		if errors.Is(err, ErrInvalidSourceData) {
+			success = false
+		} else {
+			return err
+		}
 	}
 
 	timeStamp := time.Now()
 	backupStart := timeStamp
 	if opts.TimeStamp != "" {
-		timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
+		timeStamp, err = time.ParseInLocation(global.TimeFormat, opts.TimeStamp, time.Local)
 		if err != nil {
-			return errors.Fatalf("error in time option: %v\n", err)
+			return errors.Fatalf("error in time option: %v", err)
 		}
 	}
 
-	if gopts.verbosity >= 2 && !gopts.JSON {
+	if gopts.Verbosity >= 2 && !gopts.JSON {
-		Verbosef("open repository\n")
+		printer.P("open repository")
 	}
 
-	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun)
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun, printer)
 	if err != nil {
 		return err
 	}
 	defer unlock()
 
-	var progressPrinter backup.ProgressPrinter
+	progressReporter := backup.NewProgress(printer,
-	if gopts.JSON {
+		ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()))
-		progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
-	} else {
-		progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
-	}
-	progressReporter := backup.NewProgress(progressPrinter,
-		calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 	defer progressReporter.Done()
 
 	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
-	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
+	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, printer.E)
 	if err != nil {
 		return err
 	}
 
-	// rejectFuncs collect functions that can reject items from the backup based on path and file info
+	var parentSnapshot *data.Snapshot
-	rejectFuncs, err := collectRejectFuncs(opts, targets)
-	if err != nil {
-		return err
-	}
-
-	var parentSnapshot *restic.Snapshot
 	if !opts.Stdin {
 		parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
 		if err != nil {
@@ -521,42 +544,22 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 
 		if !gopts.JSON {
 			if parentSnapshot != nil {
-				progressPrinter.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
+				printer.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
 			} else {
-				progressPrinter.P("no parent snapshot found, will read all files\n")
+				printer.P("no parent snapshot found, will read all files\n")
 			}
 		}
 	}
 
 	if !gopts.JSON {
-		progressPrinter.V("load index files")
+		printer.V("load index files")
 	}
 
-	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
+	err = repo.LoadIndex(ctx, printer)
-
-	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
-	selectByNameFilter := func(item string) bool {
-		for _, reject := range rejectByNameFuncs {
-			if reject(item) {
-				return false
-			}
-		}
-		return true
-	}
-
-	selectFilter := func(item string, fi os.FileInfo) bool {
-		for _, reject := range rejectFuncs {
-			if reject(item, fi) {
-				return false
-			}
-		}
-		return true
-	}
-
 	var targetFS fs.FS = fs.Local{}
 	if runtime.GOOS == "windows" && opts.UseFsSnapshot {
 		if err = fs.HasSufficientPrivilegesForVSS(); err != nil {
@@ -569,7 +572,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 
 		messageHandler := func(msg string, args ...interface{}) {
 			if !gopts.JSON {
-				progressPrinter.P(msg, args...)
+				printer.P(msg, args...)
 			}
 		}
 
@@ -580,21 +583,22 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 
 	if opts.Stdin || opts.StdinCommand {
 		if !gopts.JSON {
-			progressPrinter.V("read data from stdin")
+			printer.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
-		var source io.ReadCloser = os.Stdin
+		source := term.InputRaw()
 		if opts.StdinCommand {
-			source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
+			source, err = fs.NewCommandReader(ctx, args, printer.E)
 			if err != nil {
 				return err
 			}
 		}
-		targetFS = &fs.Reader{
+		targetFS, err = fs.NewReader(filename, source, fs.ReaderOptions{
-			ModTime:    timeStamp,
+			ModTime: timeStamp,
-			Name:       filename,
+			Mode:    0644,
-			Mode:       0644,
+		})
-			ReadCloser: source,
+		if err != nil {
+			return fmt.Errorf("failed to backup from stdin: %w", err)
 		}
 		targets = []string{filename}
 	}
@@ -603,6 +607,15 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		targetFS = backupFSTestHook(targetFS)
 	}
 
+	// rejectFuncs collect functions that can reject items from the backup based on path and file info
+	rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS, printer.E)
+	if err != nil {
+		return err
+	}
+
+	selectByNameFilter := archiver.CombineRejectByNames(rejectByNameFuncs)
+	selectFilter := archiver.CombineRejects(rejectFuncs)
+
 	wg, wgCtx := errgroup.WithContext(ctx)
 	cancelCtx, cancel := context.WithCancel(wgCtx)
 	defer cancel()
@@ -611,11 +624,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		sc := archiver.NewScanner(targetFS)
 		sc.SelectByName = selectByNameFilter
 		sc.Select = selectFilter
-		sc.Error = progressPrinter.ScannerError
+		sc.Error = printer.ScannerError
 		sc.Result = progressReporter.ReportTotal
 
 		if !gopts.JSON {
-			progressPrinter.V("start scan on %v", targets)
+			printer.V("start scan on %v", targets)
 		}
 		wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 	}
@@ -624,7 +637,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
-	success := true
+
 	arch.Error = func(item string, err error) error {
 		success = false
 		reterr := progressReporter.Error(item, err)
@@ -655,12 +668,12 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		Time:            timeStamp,
 		Hostname:        opts.Host,
 		ParentSnapshot:  parentSnapshot,
-		ProgramVersion:  "restic " + version,
+		ProgramVersion:  "restic " + global.Version,
 		SkipIfUnchanged: opts.SkipIfUnchanged,
 	}
 
 	if !gopts.JSON {
-		progressPrinter.V("start backup on %v", targets)
+		printer.V("start backup on %v", targets)
 	}
 	_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
 

cmd/restic/cmd_backup_integration_test.go

@@ -3,35 +3,36 @@ package main
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/data"
+	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
+func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) error {
-	return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+	return withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
 		t.Logf("backing up %v in %v", target, dir)
 		if dir != "" {
 			cleanup := rtest.Chdir(t, dir)
 			defer cleanup()
 		}
 
-		opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
+		opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true}
-		return runBackup(ctx, opts, gopts, term, target)
+		return runBackup(ctx, opts, gopts, gopts.Term, target)
 	})
 }
 
-func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
+func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) {
 	err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
-	rtest.Assert(t, err == nil, "Error while backing up")
+	rtest.Assert(t, err == nil, "Error while backing up: %v", err)
 }
 
 func TestBackup(t *testing.T) {
@@ -56,13 +57,13 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testListSnapshots(t, env.gopts, 1)
 
 	testRunCheck(t, env.gopts)
-	stat1 := dirStats(env.repo)
+	stat1 := dirStats(t, env.repo)
 
 	// second backup, implicit incremental
 	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	snapshotIDs := testListSnapshots(t, env.gopts, 2)
 
-	stat2 := dirStats(env.repo)
+	stat2 := dirStats(t, env.repo)
 	if stat2.size > stat1.size+stat1.size/10 {
 		t.Error("repository size has grown by more than 10 percent")
 	}
@@ -74,7 +75,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	snapshotIDs = testListSnapshots(t, env.gopts, 3)
 
-	stat3 := dirStats(env.repo)
+	stat3 := dirStats(t, env.repo)
 	if stat3.size > stat1.size+stat1.size/10 {
 		t.Error("repository size has grown by more than 10 percent")
 	}
@@ -85,7 +86,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
 		testRunRestore(t, env.gopts, restoredir, snapshotID.String()+":"+toPathInSnapshot(filepath.Dir(env.testdata)))
-		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
+		diff := directoriesContentsDiff(t, env.testdata, filepath.Join(restoredir, "testdata"))
 		rtest.Assert(t, diff == "", "directories are not equal: %v", diff)
 	}
 
@@ -132,7 +133,7 @@ type vssDeleteOriginalFS struct {
 	hasRemoved bool
 }
 
-func (f *vssDeleteOriginalFS) Lstat(name string) (os.FileInfo, error) {
+func (f *vssDeleteOriginalFS) Lstat(name string) (*fs.ExtendedFileInfo, error) {
 	if !f.hasRemoved {
 		// call Lstat to trigger snapshot creation
 		_, _ = f.FS.Lstat(name)
@@ -218,41 +219,41 @@ func TestDryRunBackup(t *testing.T) {
 	// dry run before first backup
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
 	snapshotIDs := testListSnapshots(t, env.gopts, 0)
-	packIDs := testRunList(t, "packs", env.gopts)
+	packIDs := testRunList(t, env.gopts, "packs")
 	rtest.Assert(t, len(packIDs) == 0,
 		"expected no data, got %v", snapshotIDs)
-	indexIDs := testRunList(t, "index", env.gopts)
+	indexIDs := testRunList(t, env.gopts, "index")
 	rtest.Assert(t, len(indexIDs) == 0,
 		"expected no index, got %v", snapshotIDs)
 
 	// first backup
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 	snapshotIDs = testListSnapshots(t, env.gopts, 1)
-	packIDs = testRunList(t, "packs", env.gopts)
+	packIDs = testRunList(t, env.gopts, "packs")
-	indexIDs = testRunList(t, "index", env.gopts)
+	indexIDs = testRunList(t, env.gopts, "index")
 
 	// dry run between backups
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
 	snapshotIDsAfter := testListSnapshots(t, env.gopts, 1)
 	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
-	dataIDsAfter := testRunList(t, "packs", env.gopts)
+	dataIDsAfter := testRunList(t, env.gopts, "packs")
 	rtest.Equals(t, packIDs, dataIDsAfter)
-	indexIDsAfter := testRunList(t, "index", env.gopts)
+	indexIDsAfter := testRunList(t, env.gopts, "index")
 	rtest.Equals(t, indexIDs, indexIDsAfter)
 
 	// second backup, implicit incremental
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
 	snapshotIDs = testListSnapshots(t, env.gopts, 2)
-	packIDs = testRunList(t, "packs", env.gopts)
+	packIDs = testRunList(t, env.gopts, "packs")
-	indexIDs = testRunList(t, "index", env.gopts)
+	indexIDs = testRunList(t, env.gopts, "index")
 
 	// another dry run
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
 	snapshotIDsAfter = testListSnapshots(t, env.gopts, 2)
 	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
-	dataIDsAfter = testRunList(t, "packs", env.gopts)
+	dataIDsAfter = testRunList(t, env.gopts, "packs")
 	rtest.Equals(t, packIDs, dataIDsAfter)
-	indexIDsAfter = testRunList(t, "index", env.gopts)
+	indexIDsAfter = testRunList(t, env.gopts, "index")
 	rtest.Equals(t, indexIDs, indexIDsAfter)
 }
 
@@ -262,22 +263,27 @@ func TestBackupNonExistingFile(t *testing.T) {
 
 	testSetupBackupData(t, env)
 
-	_ = withRestoreGlobalOptions(func() error {
+	p := filepath.Join(env.testdata, "0", "0", "9")
-		globalOptions.stderr = io.Discard
+	dirs := []string{
+		filepath.Join(p, "0"),
+		filepath.Join(p, "1"),
+		filepath.Join(p, "nonexisting"),
+		filepath.Join(p, "5"),
+	}
 
-		p := filepath.Join(env.testdata, "0", "0", "9")
+	opts := BackupOptions{}
-		dirs := []string{
-			filepath.Join(p, "0"),
-			filepath.Join(p, "1"),
-			filepath.Join(p, "nonexisting"),
-			filepath.Join(p, "5"),
-		}
 
-		opts := BackupOptions{}
+	// mix of existing and non-existing files
-
+	err := testRunBackupAssumeFailure(t, "", dirs, opts, env.gopts)
-		testRunBackup(t, "", dirs, opts, env.gopts)
+	rtest.Assert(t, err != nil, "expected error for non-existing file")
-		return nil
+	rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "expected ErrInvalidSourceData; got %v", err)
-	})
+	// only non-existing file
+	dirs = []string{
+		filepath.Join(p, "nonexisting"),
+	}
+	err = testRunBackupAssumeFailure(t, "", dirs, opts, env.gopts)
+	rtest.Assert(t, err != nil, "expected error for non-existing file")
+	rtest.Assert(t, errors.Is(err, ErrNoSourceData), "expected ErrNoSourceData; got %v", err)
 }
 
 func TestBackupSelfHealing(t *testing.T) {
@@ -365,12 +371,7 @@ func TestBackupExclude(t *testing.T) {
 	for _, filename := range backupExcludeFilenames {
 		fp := filepath.Join(datadir, filename)
 		rtest.OK(t, os.MkdirAll(filepath.Dir(fp), 0755))
-
+		rtest.OK(t, os.WriteFile(fp, []byte(filename), 0o666))
-		f, err := os.Create(fp)
-		rtest.OK(t, err)
-
-		fmt.Fprint(f, filename)
-		rtest.OK(t, f.Close())
 	}
 
 	snapshots := make(map[string]struct{})
@@ -443,13 +444,13 @@ func TestIncrementalBackup(t *testing.T) {
 
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
-	stat1 := dirStats(env.repo)
+	stat1 := dirStats(t, env.repo)
 
 	rtest.OK(t, appendRandomData(testfile, incrementalSecondWrite))
 
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
-	stat2 := dirStats(env.repo)
+	stat2 := dirStats(t, env.repo)
 	if stat2.size-stat1.size > incrementalFirstWrite {
 		t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
 	}
@@ -459,14 +460,13 @@ func TestIncrementalBackup(t *testing.T) {
 
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
-	stat3 := dirStats(env.repo)
+	stat3 := dirStats(t, env.repo)
 	if stat3.size-stat2.size > incrementalFirstWrite {
 		t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
 	}
 	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
 }
 
-// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupTags(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -486,7 +486,7 @@ func TestBackupTags(t *testing.T) {
 		"expected no tags, got %v", newest.Tags)
 	parent := newest
 
-	opts.Tags = restic.TagLists{[]string{"NL"}}
+	opts.Tags = data.TagLists{[]string{"NL"}}
 	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
 	newest, _ = testRunSnapshots(t, env.gopts)
@@ -502,7 +502,6 @@ func TestBackupTags(t *testing.T) {
 		"expected parent to be %v, got %v", parent.ID, newest.Parent)
 }
 
-// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupProgramVersion(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -514,7 +513,7 @@ func TestBackupProgramVersion(t *testing.T) {
 	if newest == nil {
 		t.Fatal("expected a backup, got nil")
 	}
-	resticVersion := "restic " + version
+	resticVersion := "restic " + global.Version
 	rtest.Assert(t, newest.ProgramVersion == resticVersion,
 		"expected %v, got %v", resticVersion, newest.ProgramVersion)
 }
@@ -572,7 +571,7 @@ func TestHardLink(t *testing.T) {
 		restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
 		t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
 		testRunRestore(t, env.gopts, restoredir, snapshotID.String())
-		diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
+		diff := directoriesContentsDiff(t, env.testdata, filepath.Join(restoredir, "testdata"))
 		rtest.Assert(t, diff == "", "directories are not equal %v", diff)
 
 		linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
@@ -637,12 +636,15 @@ func TestStdinFromCommand(t *testing.T) {
 
 	testSetupBackupData(t, env)
 	opts := BackupOptions{
-		StdinCommand:  true,
+		StdinCommand: true,
-		StdinFilename: "stdin",
+		// test that subdirectories are handled correctly
+		StdinFilename: "stdin/subdir/file",
 	}
 
 	testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts)
-	testListSnapshots(t, env.gopts, 1)
+	snapshots := testListSnapshots(t, env.gopts, 1)
+	files := testRunLs(t, env.gopts, snapshots[0].String())
+	rtest.Assert(t, includes(files, "/stdin/subdir/file"), "file %q missing from snapshot, got %v", "stdin/subdir/file", files)
 
 	testRunCheck(t, env.gopts)
 }
@@ -705,7 +707,7 @@ func TestBackupEmptyPassword(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 
-	env.gopts.password = ""
+	env.gopts.Password = ""
 	env.gopts.InsecureNoPassword = true
 
 	testSetupBackupData(t, env)

cmd/restic/cmd_backup_test.go

@@ -39,21 +39,24 @@ func TestCollectTargets(t *testing.T) {
 	f1, err := os.Create(filepath.Join(dir, "fromfile"))
 	rtest.OK(t, err)
 	// Empty lines should be ignored. A line starting with '#' is a comment.
-	fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
+	_, err = fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
+	rtest.OK(t, err)
 	rtest.OK(t, f1.Close())
 
 	f2, err := os.Create(filepath.Join(dir, "fromfile-verbatim"))
 	rtest.OK(t, err)
 	for _, filename := range []string{fooSpace, barStar} {
 		// Empty lines should be ignored. CR+LF is allowed.
-		fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
+		_, err = fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
+		rtest.OK(t, err)
 	}
 	rtest.OK(t, f2.Close())
 
 	f3, err := os.Create(filepath.Join(dir, "fromfile-raw"))
 	rtest.OK(t, err)
 	for _, filename := range []string{"baz", "quux"} {
-		fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
+		_, err = fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
+		rtest.OK(t, err)
 	}
 	rtest.OK(t, err)
 	rtest.OK(t, f3.Close())
@@ -64,10 +67,13 @@ func TestCollectTargets(t *testing.T) {
 		FilesFromRaw:      []string{f3.Name()},
 	}
 
-	targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")})
+	targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")}, t.Logf, nil)
 	rtest.OK(t, err)
 	sort.Strings(targets)
 	rtest.Equals(t, expect, targets)
+
+	_, err = collectTargets(opts, []string{filepath.Join(dir, "cmdline arg"), filepath.Join(dir, "non-existing-file")}, t.Logf, nil)
+	rtest.Assert(t, err == ErrInvalidSourceData, "expected error when not all targets exist")
 }
 
 func TestReadFilenamesRaw(t *testing.T) {

cmd/restic/cmd_cache.go

@@ -10,16 +10,20 @@ import (
 
 	"github.com/restic/restic/internal/backend/cache"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdCache = &cobra.Command{
+func newCacheCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "cache",
+	var opts CacheOptions
-	Short: "Operate on local cache directories",
+
-	Long: `
+	cmd := &cobra.Command{
+		Use:   "cache",
+		Short: "Operate on local cache directories",
+		Long: `
 The "cache" command allows listing and cleaning local cache directories.
 
 EXIT STATUS
@@ -28,11 +32,15 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	GroupID:           cmdGroupDefault,
+		GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(_ *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
-		return runCache(cacheOptions, globalOptions, args)
+			return runCache(opts, *globalOptions, args, globalOptions.Term)
-	},
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // CacheOptions bundles all options for the snapshots command.
@@ -42,18 +50,15 @@ type CacheOptions struct {
 	NoSize  bool
 }
 
-var cacheOptions CacheOptions
+func (opts *CacheOptions) AddFlags(f *pflag.FlagSet) {
-
+	f.BoolVar(&opts.Cleanup, "cleanup", false, "remove old cache directories")
-func init() {
+	f.UintVar(&opts.MaxAge, "max-age", 30, "max age in `days` for cache directories to be considered old")
-	cmdRoot.AddCommand(cmdCache)
+	f.BoolVar(&opts.NoSize, "no-size", false, "do not output the size of the cache directories")
-
-	f := cmdCache.Flags()
-	f.BoolVar(&cacheOptions.Cleanup, "cleanup", false, "remove old cache directories")
-	f.UintVar(&cacheOptions.MaxAge, "max-age", 30, "max age in `days` for cache directories to be considered old")
-	f.BoolVar(&cacheOptions.NoSize, "no-size", false, "do not output the size of the cache directories")
 }
 
-func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
+func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Terminal) error {
+	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
+
 	if len(args) > 0 {
 		return errors.Fatal("the cache command expects no arguments, only options - please see `restic help cache` for usage and flags")
 	}
@@ -81,17 +86,17 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		if len(oldDirs) == 0 {
-			Verbosef("no old cache dirs found\n")
+			printer.P("no old cache dirs found")
 			return nil
 		}
 
-		Verbosef("remove %d old cache directories\n", len(oldDirs))
+		printer.P("remove %d old cache directories", len(oldDirs))
 
 		for _, item := range oldDirs {
 			dir := filepath.Join(cachedir, item.Name())
-			err = fs.RemoveAll(dir)
+			err = os.RemoveAll(dir)
 			if err != nil {
-				Warnf("unable to remove %v: %v\n", dir, err)
+				printer.E("unable to remove %v: %v", dir, err)
 			}
 		}
 
@@ -121,7 +126,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if len(dirs) == 0 {
-		Printf("no cache dirs found, basedir is %v\n", cachedir)
+		printer.S("no cache dirs found, basedir is %v", cachedir)
 		return nil
 	}
 
@@ -157,8 +162,8 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		})
 	}
 
-	_ = tab.Write(globalOptions.stdout)
+	_ = tab.Write(gopts.Term.OutputWriter())
-	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
+	printer.S("%d cache dirs in %s", len(dirs), cachedir)
 
 	return nil
 }

cmd/restic/cmd_cat.go

@@ -7,17 +7,21 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 )
 
 var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
 
-var cmdCat = &cobra.Command{
+func newCatCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
+	cmd := &cobra.Command{
-	Short: "Print internal objects to stdout",
+		Use:   "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
-	Long: `
+		Short: "Print internal objects to stdout",
+		Long: `
 The "cat" command is used to print internal objects to stdout.
 
 EXIT STATUS
@@ -29,16 +33,14 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
+		GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		return runCat(cmd.Context(), globalOptions, args)
+			return runCat(cmd.Context(), *globalOptions, args, globalOptions.Term)
-	},
+		},
-	ValidArgs: catAllowedCmds,
+		ValidArgs: catAllowedCmds,
-}
+	}
-
+	return cmd
-func init() {
-	cmdRoot.AddCommand(cmdCat)
 }
 
 func validateCatArgs(args []string) error {
@@ -64,12 +66,14 @@ func validateCatArgs(args []string) error {
 	return nil
 }
 
-func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
+func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
+	printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
+
 	if err := validateCatArgs(args); err != nil {
 		return err
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
 		return err
 	}
@@ -81,7 +85,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 	if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
 		id, err = restic.ParseID(args[1])
 		if err != nil {
-			return errors.Fatalf("unable to parse ID: %v\n", err)
+			return errors.Fatalf("unable to parse ID: %v", err)
 		}
 	}
 
@@ -92,7 +96,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		Println(string(buf))
+		printer.S(string(buf))
 		return nil
 	case "index":
 		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
@@ -100,12 +104,12 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		Println(string(buf))
+		printer.S(string(buf))
 		return nil
 	case "snapshot":
-		sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
+		sn, _, err := data.FindSnapshot(ctx, repo, repo, args[1])
 		if err != nil {
-			return errors.Fatalf("could not find snapshot: %v\n", err)
+			return errors.Fatalf("could not find snapshot: %v", err)
 		}
 
 		buf, err := json.MarshalIndent(sn, "", "  ")
@@ -113,7 +117,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		Println(string(buf))
+		printer.S(string(buf))
 		return nil
 	case "key":
 		key, err := repository.LoadKey(ctx, repo, id)
@@ -126,7 +130,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		Println(string(buf))
+		printer.S(string(buf))
 		return nil
 	case "masterkey":
 		buf, err := json.MarshalIndent(repo.Key(), "", "  ")
@@ -134,7 +138,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		Println(string(buf))
+		printer.S(string(buf))
 		return nil
 	case "lock":
 		lock, err := restic.LoadLock(ctx, repo, id)
@@ -147,7 +151,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		Println(string(buf))
+		printer.S(string(buf))
 		return nil
 
 	case "pack":
@@ -159,15 +163,14 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 
 		hash := restic.Hash(buf)
 		if !hash.Equal(id) {
-			Warnf("Warning: hash of data does not match ID, want\n  %v\ngot:\n  %v\n", id.String(), hash.String())
+			printer.E("Warning: hash of data does not match ID, want\n  %v\ngot:\n  %v", id.String(), hash.String())
 		}
 
-		_, err = globalOptions.stdout.Write(buf)
+		_, err = term.OutputRaw().Write(buf)
 		return err
 
 	case "blob":
-		bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+		err = repo.LoadIndex(ctx, printer)
-		err = repo.LoadIndex(ctx, bar)
 		if err != nil {
 			return err
 		}
@@ -182,25 +185,24 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 				return err
 			}
 
-			_, err = globalOptions.stdout.Write(buf)
+			_, err = term.OutputRaw().Write(buf)
 			return err
 		}
 
 		return errors.Fatal("blob not found")
 
 	case "tree":
-		sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
+		sn, subfolder, err := data.FindSnapshot(ctx, repo, repo, args[1])
 		if err != nil {
-			return errors.Fatalf("could not find snapshot: %v\n", err)
+			return errors.Fatalf("could not find snapshot: %v", err)
 		}
 
-		bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+		err = repo.LoadIndex(ctx, printer)
-		err = repo.LoadIndex(ctx, bar)
 		if err != nil {
 			return err
 		}
 
-		sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+		sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
 		if err != nil {
 			return err
 		}
@@ -209,7 +211,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		if err != nil {
 			return err
 		}
-		_, err = globalOptions.stdout.Write(buf)
+		_, err = term.OutputRaw().Write(buf)
 		return err
 
 	default:

cmd/restic/cmd_check.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"math/rand"
 	"os"
 	"strconv"
@@ -10,22 +11,25 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
 	"github.com/restic/restic/internal/backend/cache"
 	"github.com/restic/restic/internal/checker"
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/progress"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-var cmdCheck = &cobra.Command{
+func newCheckCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "check [flags]",
+	var opts CheckOptions
-	Short: "Check the repository for errors",
+	cmd := &cobra.Command{
-	Long: `
+		Use:   "check [flags]",
+		Short: "Check the repository for errors",
+		Long: `
 The "check" command tests the repository for errors and reports any errors it
 finds. It can also be used to read all data and therefore simulate a restore.
 
@@ -41,16 +45,26 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
+		GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		term, cancel := setupTermstatus()
+			finalizeSnapshotFilter(&opts.SnapshotFilter)
-		defer cancel()
+			summary, err := runCheck(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
-		return runCheck(cmd.Context(), checkOptions, globalOptions, args, term)
+			if globalOptions.JSON {
-	},
+				if err != nil && summary.NumErrors == 0 {
-	PreRunE: func(_ *cobra.Command, _ []string) error {
+					summary.NumErrors = 1
-		return checkFlags(checkOptions)
+				}
-	},
+				globalOptions.Term.Print(ui.ToJSONString(summary))
+			}
+			return err
+		},
+		PreRunE: func(_ *cobra.Command, _ []string) error {
+			return checkFlags(opts)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // CheckOptions bundles all options for the 'check' command.
@@ -59,16 +73,12 @@ type CheckOptions struct {
 	ReadDataSubset string
 	CheckUnused    bool
 	WithCache      bool
+	data.SnapshotFilter
 }
 
-var checkOptions CheckOptions
+func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
-
+	f.BoolVar(&opts.ReadData, "read-data", false, "read all data blobs")
-func init() {
+	f.StringVar(&opts.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
-	cmdRoot.AddCommand(cmdCheck)
-
-	f := cmdCheck.Flags()
-	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
 	var ignored bool
 	f.BoolVar(&ignored, "check-unused", false, "find unused blobs")
 	err := f.MarkDeprecated("check-unused", "`--check-unused` is deprecated and will be ignored")
@@ -76,7 +86,8 @@ func init() {
 		// MarkDeprecated only returns an error when the flag is not found
 		panic(err)
 	}
-	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
+	f.BoolVar(&opts.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
 }
 
 func checkFlags(opts CheckOptions) error {
@@ -164,7 +175,7 @@ func parsePercentage(s string) (float64, error) {
 //   - if the user explicitly requested --no-cache, we don't use any cache
 //   - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
 //   - by default, we use a cache in a temporary directory that is deleted after the check
-func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress.Printer) (cleanup func()) {
+func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progress.Printer) (cleanup func()) {
 	cleanup = func() {}
 	if opts.WithCache {
 		// use the default cache, no setup needed
@@ -185,7 +196,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
 		// use a cache in a temporary directory
 		err := os.MkdirAll(cachedir, 0755)
 		if err != nil {
-			Warnf("unable to create cache directory %s, disabling cache: %v\n", cachedir, err)
+			printer.E("unable to create cache directory %s, disabling cache: %v", cachedir, err)
 			gopts.NoCache = true
 			return cleanup
 		}
@@ -202,7 +213,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
 	printer.P("using temporary cache in %v\n", tempdir)
 
 	cleanup = func() {
-		err := fs.RemoveAll(tempdir)
+		err := os.RemoveAll(tempdir)
 		if err != nil {
 			printer.E("error removing temporary cache directory: %v\n", err)
 		}
@@ -211,12 +222,15 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress
 	return cleanup
 }
 
-func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error {
+func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args []string, term ui.Terminal) (checkSummary, error) {
-	if len(args) != 0 {
+	summary := checkSummary{MessageType: "summary"}
-		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
-	}
 
-	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+	var printer progress.Printer
+	if !gopts.JSON {
+		printer = ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
+	} else {
+		printer = newJSONErrorPrinter(term)
+	}
 
 	cleanup := prepareCheckCache(opts, &gopts, printer)
 	defer cleanup()
@@ -224,55 +238,44 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	if !gopts.NoLock {
 		printer.P("create exclusive lock for repository\n")
 	}
-	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
-		return err
+		return summary, err
 	}
 	defer unlock()
 
 	chkr := checker.New(repo, opts.CheckUnused)
-	err = chkr.LoadSnapshots(ctx)
+	err = chkr.LoadSnapshots(ctx, &opts.SnapshotFilter, args)
 	if err != nil {
-		return err
+		return summary, err
 	}
 
 	printer.P("load indexes\n")
-	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
+	hints, errs := chkr.LoadIndex(ctx, printer)
-	hints, errs := chkr.LoadIndex(ctx, bar)
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	errorsFound := false
-	suggestIndexRebuild := false
-	suggestLegacyIndexRebuild := false
-	mixedFound := false
 	for _, hint := range hints {
 		switch hint.(type) {
-		case *checker.ErrDuplicatePacks:
+		case *repository.ErrDuplicatePacks:
-			term.Print(hint.Error())
+			printer.S("%s", hint.Error())
-			suggestIndexRebuild = true
+			summary.HintRepairIndex = true
-		case *checker.ErrOldIndexFormat:
+		case *repository.ErrMixedPack:
-			printer.E("error: %v\n", hint)
+			printer.S("%s", hint.Error())
-			suggestLegacyIndexRebuild = true
+			summary.HintPrune = true
-			errorsFound = true
-		case *checker.ErrMixedPack:
-			term.Print(hint.Error())
-			mixedFound = true
 		default:
 			printer.E("error: %v\n", hint)
 			errorsFound = true
 		}
 	}
 
-	if suggestIndexRebuild {
+	if summary.HintRepairIndex {
-		term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
+		printer.S("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
 	}
-	if suggestLegacyIndexRebuild {
+	if summary.HintPrune {
-		printer.E("error: Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
+		printer.S("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
-	}
-	if mixedFound {
-		term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
 	}
 
 	if len(errs) > 0 {
@@ -280,8 +283,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 			printer.E("error: %v\n", err)
 		}
 
+		summary.NumErrors += len(errs)
+		summary.HintRepairIndex = true
 		printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n")
-		return errors.Fatal("repository contains errors")
+		return summary, errors.Fatal("repository contains errors")
 	}
 
 	orphanedPacks := 0
@@ -292,7 +297,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	go chkr.Packs(ctx, errChan)
 
 	for err := range errChan {
-		var packErr *checker.PackError
+		var packErr *repository.PackError
 		if errors.As(err, &packErr) {
 			if packErr.Orphaned {
 				orphanedPacks++
@@ -302,23 +307,24 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 					salvagePacks.Insert(packErr.ID)
 				}
 				errorsFound = true
+				summary.NumErrors++
 				printer.E("%v\n", err)
 			}
-		} else if err == checker.ErrLegacyLayout {
-			errorsFound = true
-			printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n")
 		} else {
 			errorsFound = true
 			printer.E("%v\n", err)
 		}
 	}
 
-	if orphanedPacks > 0 && !errorsFound {
+	if orphanedPacks > 0 {
-		// hide notice if repository is damaged
+		summary.HintPrune = true
-		printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
+		if !errorsFound {
+			// hide notice if repository is damaged
+			printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
+		}
 	}
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	printer.P("check snapshots, trees and blobs\n")
@@ -328,7 +334,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		bar := newTerminalProgressMax(!gopts.Quiet, 0, "snapshots", term)
+		bar := printer.NewCounter("snapshots")
 		defer bar.Done()
 		chkr.Structure(ctx, bar, errChan)
 	}()
@@ -338,9 +344,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		if e, ok := err.(*checker.TreeError); ok {
 			printer.E("error for tree %v:\n", e.ID.Str())
 			for _, treeErr := range e.Errors {
+				summary.NumErrors++
 				printer.E("  %v\n", treeErr)
 			}
 		} else {
+			summary.NumErrors++
 			printer.E("error: %v\n", err)
 		}
 	}
@@ -350,13 +358,14 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	// deadlocking in the case of errors.
 	wg.Wait()
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
+	// the following block only used for tests
 	if opts.CheckUnused {
 		unused, err := chkr.UnusedBlobs(ctx)
 		if err != nil {
-			return err
+			return summary, err
 		}
 		for _, id := range unused {
 			printer.P("unused blob %v\n", id)
@@ -364,16 +373,20 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		}
 	}
 
-	doReadData := func(packs map[restic.ID]int64) {
+	readDataFilter, err := buildPacksFilter(opts, printer, chkr.IsFiltered())
-		packCount := uint64(len(packs))
+	if err != nil {
+		return summary, err
+	}
 
-		p := newTerminalProgressMax(!gopts.Quiet, packCount, "packs", term)
+	if readDataFilter != nil {
+		p := printer.NewCounter("packs")
 		errChan := make(chan error)
 
-		go chkr.ReadPacks(ctx, packs, p, errChan)
+		go chkr.ReadPacks(ctx, readDataFilter, p, errChan)
 
 		for err := range errChan {
 			errorsFound = true
+			summary.NumErrors++
 			printer.E("%v\n", err)
 			if err, ok := err.(*repository.ErrPackData); ok {
 				salvagePacks.Insert(err.PackID)
@@ -382,70 +395,85 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		p.Done()
 	}
 
-	switch {
-	case opts.ReadData:
-		printer.P("read all data\n")
-		doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
-	case opts.ReadDataSubset != "":
-		var packs map[restic.ID]int64
-		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		if err == nil {
-			bucket := dataSubset[0]
-			totalBuckets := dataSubset[1]
-			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
-			packCount := uint64(len(packs))
-			printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
-		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
-			percentage, err := parsePercentage(opts.ReadDataSubset)
-			if err == nil {
-				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
-				printer.P("read %.1f%% of data packs\n", percentage)
-			}
-		} else {
-			repoSize := int64(0)
-			allPacks := chkr.GetPacks()
-			for _, size := range allPacks {
-				repoSize += size
-			}
-			if repoSize == 0 {
-				return errors.Fatal("Cannot read from a repository having size 0")
-			}
-			subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
-			if subsetSize > repoSize {
-				subsetSize = repoSize
-			}
-			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
-			printer.P("read %d bytes of data packs\n", subsetSize)
-		}
-		if packs == nil {
-			return errors.Fatal("internal error: failed to select packs to check")
-		}
-		doReadData(packs)
-	}
-
 	if len(salvagePacks) > 0 {
 		printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
-		var strIDs []string
 		for id := range salvagePacks {
-			strIDs = append(strIDs, id.String())
+			summary.BrokenPacks = append(summary.BrokenPacks, id.String())
 		}
-		printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
+		printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(summary.BrokenPacks, " "))
 		printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
 	}
 
 	if ctx.Err() != nil {
-		return ctx.Err()
+		return summary, ctx.Err()
 	}
 
 	if errorsFound {
 		if len(salvagePacks) == 0 {
 			printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n")
 		}
-		return errors.Fatal("repository contains errors")
+		return summary, errors.Fatal("repository contains errors")
 	}
 	printer.P("no errors were found\n")
+	return summary, nil
+}
 
-	return nil
+func buildPacksFilter(opts CheckOptions, printer progress.Printer,
+	filteredStatus bool) (func(packs map[restic.ID]int64) map[restic.ID]int64, error) {
+	typeData := ""
+	if filteredStatus {
+		typeData = "filtered "
+	}
+	switch {
+	case opts.ReadData:
+		return func(packs map[restic.ID]int64) map[restic.ID]int64 {
+			printer.P("read all %sdata", typeData)
+			return packs
+		}, nil
+	case opts.ReadDataSubset != "":
+		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
+		if err == nil {
+			bucket := dataSubset[0]
+			totalBuckets := dataSubset[1]
+			return func(packs map[restic.ID]int64) map[restic.ID]int64 {
+				packCount := uint64(len(packs))
+				packs = selectPacksByBucket(packs, bucket, totalBuckets)
+				printer.P("read group #%d of %d %sdata packs (out of total %d packs in %d groups", bucket, len(packs), typeData, packCount, totalBuckets)
+				return packs
+			}, nil
+		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
+			percentage, err := parsePercentage(opts.ReadDataSubset)
+			if err != nil {
+				return nil, err
+			}
+			return func(packs map[restic.ID]int64) map[restic.ID]int64 {
+				printer.P("read %.1f%% of %spackfiles", percentage, typeData)
+				return selectRandomPacksByPercentage(packs, percentage)
+			}, nil
+		}
+
+		repoSize := int64(0)
+		return func(packs map[restic.ID]int64) map[restic.ID]int64 {
+			for _, size := range packs {
+				repoSize += size
+			}
+			subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
+			if subsetSize > repoSize {
+				subsetSize = repoSize
+			}
+			if repoSize > 0 {
+				packs = selectRandomPacksByFileSize(packs, subsetSize, repoSize)
+			}
+			percentage := float64(subsetSize) / float64(repoSize) * 100.0
+			if repoSize == 0 {
+				percentage = 100
+			}
+			printer.P("read %d bytes (%.1f%%) of %sdata packs\n", subsetSize, percentage, typeData)
+			return packs
+		}, nil
+	}
+
+	return nil, nil
 }
 
 // selectPacksByBucket selects subsets of packs by ranges of buckets.
@@ -491,3 +519,47 @@ func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64,
 	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
 	return packs
 }
+
+type checkSummary struct {
+	MessageType     string   `json:"message_type"` // "summary"
+	NumErrors       int      `json:"num_errors"`
+	BrokenPacks     []string `json:"broken_packs"`         // run "restic repair packs ID..." and "restic repair snapshots --forget" to remove damaged files
+	HintRepairIndex bool     `json:"suggest_repair_index"` // run "restic repair index"
+	HintPrune       bool     `json:"suggest_prune"`        // run "restic prune"
+}
+
+type checkError struct {
+	MessageType string `json:"message_type"` // "error"
+	Message     string `json:"message"`
+}
+
+type jsonErrorPrinter struct {
+	term ui.Terminal
+}
+
+func newJSONErrorPrinter(term ui.Terminal) *jsonErrorPrinter {
+	return &jsonErrorPrinter{
+		term: term,
+	}
+}
+
+func (*jsonErrorPrinter) NewCounter(_ string) *progress.Counter {
+	return nil
+}
+
+func (*jsonErrorPrinter) NewCounterTerminalOnly(_ string) *progress.Counter {
+	return nil
+}
+
+func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
+	status := checkError{
+		MessageType: "error",
+		Message:     fmt.Sprintf(msg, args...),
+	}
+	p.term.Error(ui.ToJSONString(status))
+}
+func (*jsonErrorPrinter) S(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) P(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) PT(_ string, _ ...interface{}) {}
+func (*jsonErrorPrinter) V(_ string, _ ...interface{})  {}
+func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {}

cmd/restic/cmd_check_integration_test.go

@@ -1,38 +1,101 @@
 package main
 
 import (
-	"bytes"
 	"context"
+	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/global"
 	rtest "github.com/restic/restic/internal/test"
-	"github.com/restic/restic/internal/ui/termstatus"
 )
 
-func testRunCheck(t testing.TB, gopts GlobalOptions) {
+func testRunCheck(t testing.TB, gopts global.Options) {
 	t.Helper()
-	output, err := testRunCheckOutput(gopts, true)
+	output, err := testRunCheckOutput(t, gopts, true)
 	if err != nil {
 		t.Error(output)
 		t.Fatalf("unexpected error: %+v", err)
 	}
 }
 
-func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
+func testRunCheckMustFail(t testing.TB, gopts global.Options) {
 	t.Helper()
-	_, err := testRunCheckOutput(gopts, false)
+	_, err := testRunCheckOutput(t, gopts, false)
 	rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository")
 }
 
-func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
+func testRunCheckOutput(t testing.TB, gopts global.Options, checkUnused bool) (string, error) {
-	buf := bytes.NewBuffer(nil)
+	buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
-	gopts.stdout = buf
-	err := withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
 		opts := CheckOptions{
 			ReadData:    true,
 			CheckUnused: checkUnused,
 		}
-		return runCheck(context.TODO(), opts, gopts, nil, term)
+		_, err := runCheck(context.TODO(), opts, gopts, nil, gopts.Term)
+		return err
 	})
 	return buf.String(), err
 }
+
+func testRunCheckOutputWithOpts(t testing.TB, gopts global.Options, opts CheckOptions, args []string) (string, error) {
+	buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
+		gopts.Verbosity = 2
+		_, err := runCheck(context.TODO(), opts, gopts, args, gopts.Term)
+		return err
+	})
+	return buf.String(), err
+}
+
+func TestCheckWithSnaphotFilter(t *testing.T) {
+	testCases := []struct {
+		opts           CheckOptions
+		args           []string
+		expectedOutput string
+	}{
+		{ // full --read-data, all snapshots
+			CheckOptions{ReadData: true},
+			nil,
+			"4 / 4 packs",
+		},
+		{ // full --read-data, all snapshots
+			CheckOptions{ReadData: true},
+			nil,
+			"2 / 2 snapshots",
+		},
+		{ // full --read-data, latest snapshot
+			CheckOptions{ReadData: true},
+			[]string{"latest"},
+			"2 / 2 packs",
+		},
+		{ // full --read-data, latest snapshot
+			CheckOptions{ReadData: true},
+			[]string{"latest"},
+			"1 / 1 snapshots",
+		},
+		{ // --read-data-subset, latest snapshot
+			CheckOptions{ReadDataSubset: "1%"},
+			[]string{"latest"},
+			"1 / 1 packs",
+		},
+		{ // --read-data-subset, latest snapshot
+			CheckOptions{ReadDataSubset: "1%"},
+			[]string{"latest"},
+			"filtered",
+		},
+	}
+
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts)
+	testRunBackup(t, env.testdata+"/0", []string{"0/9"}, opts, env.gopts)
+
+	for _, testCase := range testCases {
+		output, err := testRunCheckOutputWithOpts(t, env.gopts, testCase.opts, testCase.args)
+		rtest.OK(t, err)
+
+		hasOutput := strings.Contains(output, testCase.expectedOutput)
+		rtest.Assert(t, hasOutput, `expected to find substring %q, but did not find it`, testCase.expectedOutput)
+	}
+}

cmd/restic/cmd_check_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 	"github.com/restic/restic/internal/ui/progress"
@@ -202,7 +203,7 @@ func TestPrepareCheckCache(t *testing.T) {
 				err := os.Remove(tmpDirBase)
 				rtest.OK(t, err)
 			}
-			gopts := GlobalOptions{CacheDir: tmpDirBase}
+			gopts := global.Options{CacheDir: tmpDirBase}
 			cleanup := prepareCheckCache(testCase.opts, &gopts, &progress.NoopPrinter{})
 			files, err := os.ReadDir(tmpDirBase)
 			rtest.OK(t, err)
@@ -232,7 +233,7 @@ func TestPrepareCheckCache(t *testing.T) {
 }
 
 func TestPrepareDefaultCheckCache(t *testing.T) {
-	gopts := GlobalOptions{CacheDir: ""}
+	gopts := global.Options{CacheDir: ""}
 	cleanup := prepareCheckCache(CheckOptions{}, &gopts, &progress.NoopPrinter{})
 	_, err := os.ReadDir(gopts.CacheDir)
 	rtest.OK(t, err)

cmd/restic/cmd_copy.go

@@ -3,20 +3,29 @@ package main
 import (
 	"context"
 	"fmt"
+	"iter"
+	"sync"
+	"time"
 
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
-	"golang.org/x/sync/errgroup"
+	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/progress"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdCopy = &cobra.Command{
+func newCopyCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "copy [flags] [snapshotID ...]",
+	var opts CopyOptions
-	Short: "Copy snapshots from one repository to another",
+	cmd := &cobra.Command{
-	Long: `
+		Use:   "copy [flags] [snapshotID ...]",
+		Short: "Copy snapshots from one repository to another",
+		Long: `
 The "copy" command copies one or more snapshots from one repository to another.
 
 NOTE: This process will have to both download (read) and upload (write) the
@@ -40,31 +49,65 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
+		GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		return runCopy(cmd.Context(), copyOptions, globalOptions, args)
+			finalizeSnapshotFilter(&opts.SnapshotFilter)
-	},
+			return runCopy(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // CopyOptions bundles all options for the copy command.
 type CopyOptions struct {
-	secondaryRepoOptions
+	global.SecondaryRepoOptions
-	restic.SnapshotFilter
+	data.SnapshotFilter
 }
 
-var copyOptions CopyOptions
+func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) {
-
+	opts.SecondaryRepoOptions.AddFlags(f, "destination", "to copy snapshots from")
-func init() {
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
-	cmdRoot.AddCommand(cmdCopy)
-
-	f := cmdCopy.Flags()
-	initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
-	initMultiSnapshotFilter(f, &copyOptions.SnapshotFilter, true)
 }
 
-func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
+// collectAllSnapshots: select all snapshot trees to be copied
-	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
+func collectAllSnapshots(ctx context.Context, opts CopyOptions,
+	srcSnapshotLister restic.Lister, srcRepo restic.Repository,
+	dstSnapshotByOriginal map[restic.ID][]*data.Snapshot, args []string, printer progress.Printer,
+) iter.Seq[*data.Snapshot] {
+	return func(yield func(*data.Snapshot) bool) {
+		for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args, printer) {
+			// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
+			srcOriginal := *sn.ID()
+			if sn.Original != nil {
+				srcOriginal = *sn.Original
+			}
+			if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
+				isCopy := false
+				for _, originalSn := range originalSns {
+					if similarSnapshots(originalSn, sn) {
+						printer.V("\n%v", sn)
+						printer.V("skipping source snapshot %s, was already copied to snapshot %s", sn.ID().Str(), originalSn.ID().Str())
+						isCopy = true
+						break
+					}
+				}
+				if isCopy {
+					continue
+				}
+			}
+			if !yield(sn) {
+				return
+			}
+		}
+	}
+}
+
+func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args []string, term ui.Terminal) error {
+	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
+	secondaryGopts, isFromRepo, err := opts.SecondaryRepoOptions.FillGlobalOpts(ctx, gopts, "destination")
 	if err != nil {
 		return err
 	}
@@ -73,13 +116,13 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		gopts, secondaryGopts = secondaryGopts, gopts
 	}
 
-	ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
 		return err
 	}
 	defer unlock()
 
-	ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false)
+	ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false, printer)
 	if err != nil {
 		return err
 	}
@@ -96,18 +139,16 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 	}
 
 	debug.Log("Loading source index")
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err := srcRepo.LoadIndex(ctx, printer); err != nil {
-	if err := srcRepo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
-	bar = newIndexProgress(gopts.Quiet, gopts.JSON)
 	debug.Log("Loading destination index")
-	if err := dstRepo.LoadIndex(ctx, bar); err != nil {
+	if err := dstRepo.LoadIndex(ctx, printer); err != nil {
 		return err
 	}
 
-	dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
+	dstSnapshotByOriginal := make(map[restic.ID][]*data.Snapshot)
-	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) {
+	for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil, printer) {
 		if sn.Original != nil && !sn.Original.IsNull() {
 			dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
 		}
@@ -118,53 +159,16 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		return ctx.Err()
 	}
 
-	// remember already processed trees across all snapshots
+	selectedSnapshots := collectAllSnapshots(ctx, opts, srcSnapshotLister, srcRepo, dstSnapshotByOriginal, args, printer)
-	visitedTrees := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) {
+	if err := copyTreeBatched(ctx, srcRepo, dstRepo, selectedSnapshots, printer); err != nil {
-		// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
+		return err
-		srcOriginal := *sn.ID()
-		if sn.Original != nil {
-			srcOriginal = *sn.Original
-		}
-
-		if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
-			isCopy := false
-			for _, originalSn := range originalSns {
-				if similarSnapshots(originalSn, sn) {
-					Verboseff("\n%v\n", sn)
-					Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
-					isCopy = true
-					break
-				}
-			}
-			if isCopy {
-				continue
-			}
-		}
-		Verbosef("\n%v\n", sn)
-		Verbosef("  copy started, this may take a while...\n")
-		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
-			return err
-		}
-		debug.Log("tree copied")
-
-		// save snapshot
-		sn.Parent = nil // Parent does not have relevance in the new repo.
-		// Use Original as a persistent snapshot ID
-		if sn.Original == nil {
-			sn.Original = sn.ID()
-		}
-		newID, err := restic.SaveSnapshot(ctx, dstRepo, sn)
-		if err != nil {
-			return err
-		}
-		Verbosef("snapshot %s saved\n", newID.Str())
 	}
+
 	return ctx.Err()
 }
 
-func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
+func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool {
 	// everything except Parent and Original must match
 	if !sna.Time.Equal(snb.Time) || !sna.Tree.Equal(*snb.Tree) || sna.Hostname != snb.Hostname ||
 		sna.Username != snb.Username || sna.UID != snb.UID || sna.GID != snb.GID ||
@@ -183,64 +187,158 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 
-func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
+// copyTreeBatched copies multiple snapshots in one go. Snapshots are written after
-	visitedTrees restic.IDSet, rootTreeID restic.ID, quiet bool) error {
+// data equivalent to at least 10 packfiles was written.
+func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
+	selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error {
 
-	wg, wgCtx := errgroup.WithContext(ctx)
+	// remember already processed trees across all snapshots
+	visitedTrees := srcRepo.NewAssociatedBlobSet()
 
-	treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
+	targetSize := uint64(dstRepo.PackSize()) * 100
-		visited := visitedTrees.Has(treeID)
+	minDuration := 1 * time.Minute
-		visitedTrees.Insert(treeID)
-		return visited
-	}, nil)
 
-	copyBlobs := restic.NewBlobSet()
+	// use pull-based iterator to allow iteration in multiple steps
-	packList := restic.NewIDSet()
+	next, stop := iter.Pull(selectedSnapshots)
+	defer stop()
 
-	enqueue := func(h restic.BlobHandle) {
+	for {
-		pb := srcRepo.LookupBlob(h.Type, h.ID)
+		var batch []*data.Snapshot
-		copyBlobs.Insert(h)
+		batchSize := uint64(0)
-		for _, p := range pb {
+		startTime := time.Now()
-			packList.Insert(p.PackID)
+
+		// call WithBlobUploader() once and then loop over all selectedSnapshots
+		err := dstRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
+			for batchSize < targetSize || time.Since(startTime) < minDuration {
+				sn, ok := next()
+				if !ok {
+					break
+				}
+
+				batch = append(batch, sn)
+
+				printer.P("\n%v", sn)
+				printer.P("  copy started, this may take a while...")
+				sizeBlobs, err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, printer, uploader)
+				if err != nil {
+					return err
+				}
+				debug.Log("tree copied")
+				batchSize += sizeBlobs
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		// if no snapshots were processed in this batch, we're done
+		if len(batch) == 0 {
+			break
+		}
+
+		// add a newline to separate saved snapshot messages from the other messages
+		if len(batch) > 1 {
+			printer.P("")
+		}
+		// save all the snapshots
+		for _, sn := range batch {
+			err := copySaveSnapshot(ctx, sn, dstRepo, printer)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
-	wg.Go(func() error {
+	return nil
-		for tree := range treeStream {
+}
-			if tree.Error != nil {
-				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
-			}
 
-			// Do we already have this tree blob?
+func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
-			treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
+	visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) {
-			if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok {
-				// copy raw tree bytes to avoid problems if the serialization changes
-				enqueue(treeHandle)
-			}
 
-			for _, entry := range tree.Nodes {
+	copyBlobs := srcRepo.NewAssociatedBlobSet()
-				// Recursion into directories is handled by StreamTrees
+	packList := restic.NewIDSet()
-				// Copy the blobs for this file.
+	var lock sync.Mutex
-				for _, blobID := range entry.Content {
+
-					h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
+	enqueue := func(h restic.BlobHandle) {
-					if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
+		lock.Lock()
-						enqueue(h)
+		defer lock.Unlock()
-					}
+		if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
-				}
+			pb := srcRepo.LookupBlob(h.Type, h.ID)
+			copyBlobs.Insert(h)
+			for _, p := range pb {
+				packList.Insert(p.PackID)
+			}
+		}
+	}
+
+	err := data.StreamTrees(ctx, srcRepo, restic.IDs{rootTreeID}, nil, func(treeID restic.ID) bool {
+		handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}
+		visited := visitedTrees.Has(handle)
+		visitedTrees.Insert(handle)
+		return visited
+	}, func(treeID restic.ID, err error, nodes data.TreeNodeIterator) error {
+		if err != nil {
+			return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
+		}
+
+		// copy raw tree bytes to avoid problems if the serialization changes
+		enqueue(restic.BlobHandle{ID: treeID, Type: restic.TreeBlob})
+
+		for item := range nodes {
+			if item.Error != nil {
+				return item.Error
+			}
+			// Recursion into directories is handled by StreamTrees
+			// Copy the blobs for this file.
+			for _, blobID := range item.Node.Content {
+				enqueue(restic.BlobHandle{Type: restic.DataBlob, ID: blobID})
 			}
 		}
 		return nil
 	})
-	err := wg.Wait()
+	if err != nil {
+		return 0, err
+	}
+
+	sizeBlobs := copyStats(srcRepo, copyBlobs, packList, printer)
+	bar := printer.NewCounter("packs copied")
+	err = repository.CopyBlobs(ctx, srcRepo, dstRepo, uploader, packList, copyBlobs, bar, printer.P)
+	if err != nil {
+		return 0, errors.Fatalf("%s", err)
+	}
+	return sizeBlobs, nil
+}
+
+// copyStats: print statistics for the blobs to be copied
+func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, packList restic.IDSet, printer progress.Printer) uint64 {
+	// count and size
+	countBlobs := 0
+	sizeBlobs := uint64(0)
+	for blob := range copyBlobs.Keys() {
+		for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) {
+			countBlobs++
+			sizeBlobs += uint64(blob.Length)
+			break
+		}
+	}
+
+	printer.V("  copy %d blobs with disk size %s in %d packfiles\n",
+		countBlobs, ui.FormatBytes(uint64(sizeBlobs)), len(packList))
+	return sizeBlobs
+}
+
+func copySaveSnapshot(ctx context.Context, sn *data.Snapshot, dstRepo restic.Repository, printer progress.Printer) error {
+	sn.Parent = nil // Parent does not have relevance in the new repo.
+	// Use Original as a persistent snapshot ID
+	if sn.Original == nil {
+		sn.Original = sn.ID()
+	}
+	newID, err := data.SaveSnapshot(ctx, dstRepo, sn)
 	if err != nil {
 		return err
 	}
-
+	printer.P("snapshot %s saved, copied from source snapshot %s", newID.Str(), sn.ID().Str())
-	bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
-	_, err = repository.Repack(ctx, srcRepo, dstRepo, packList, copyBlobs, bar)
-	bar.Done()
-	if err != nil {
-		return errors.Fatal(err.Error())
-	}
 	return nil
 }

cmd/restic/cmd_copy_integration_test.go

@@ -6,23 +6,28 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/restic/restic/internal/global"
+	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui"
 )
 
-func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
+func testRunCopy(t testing.TB, srcGopts global.Options, dstGopts global.Options) {
 	gopts := srcGopts
 	gopts.Repo = dstGopts.Repo
-	gopts.password = dstGopts.password
+	gopts.Password = dstGopts.Password
 	gopts.InsecureNoPassword = dstGopts.InsecureNoPassword
 	copyOpts := CopyOptions{
-		secondaryRepoOptions: secondaryRepoOptions{
+		SecondaryRepoOptions: global.SecondaryRepoOptions{
 			Repo:               srcGopts.Repo,
-			password:           srcGopts.password,
+			Password:           srcGopts.Password,
 			InsecureNoPassword: srcGopts.InsecureNoPassword,
 		},
 	}
 
-	rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
+	rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
+		return runCopy(context.TODO(), copyOpts, gopts, nil, gopts.Term)
+	}))
 }
 
 func TestCopy(t *testing.T) {
@@ -45,8 +50,8 @@ func TestCopy(t *testing.T) {
 	copiedSnapshotIDs := testListSnapshots(t, env2.gopts, 3)
 
 	// Check that the copies size seems reasonable
-	stat := dirStats(env.repo)
+	stat := dirStats(t, env.repo)
-	stat2 := dirStats(env2.repo)
+	stat2 := dirStats(t, env2.repo)
 	sizeDiff := int64(stat.size) - int64(stat2.size)
 	if sizeDiff < 0 {
 		sizeDiff = -sizeDiff
@@ -69,7 +74,7 @@ func TestCopy(t *testing.T) {
 		testRunRestore(t, env2.gopts, restoredir, snapshotID.String())
 		foundMatch := false
 		for cmpdir := range origRestores {
-			diff := directoriesContentsDiff(restoredir, cmpdir)
+			diff := directoriesContentsDiff(t, restoredir, cmpdir)
 			if diff == "" {
 				delete(origRestores, cmpdir)
 				foundMatch = true
@@ -80,6 +85,41 @@ func TestCopy(t *testing.T) {
 	}
 
 	rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots")
+
+	// check that snapshots were properly batched while copying
+	_, _, countBlobs := testPackAndBlobCounts(t, env.gopts)
+	countTreePacksDst, countDataPacksDst, countBlobsDst := testPackAndBlobCounts(t, env2.gopts)
+
+	rtest.Equals(t, countBlobs, countBlobsDst, "expected blob count in boths repos to be equal")
+	rtest.Equals(t, countTreePacksDst, 1, "expected 1 tree packfile")
+	rtest.Equals(t, countDataPacksDst, 1, "expected 1 data packfile")
+}
+
+func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks int, countDataPacks int, countBlobs int) {
+	rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
+		printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
+		_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
+		rtest.OK(t, err)
+		defer unlock()
+
+		rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
+			blobs, _, err := repo.ListPack(context.TODO(), id, size)
+			rtest.OK(t, err)
+			rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
+
+			switch blobs[0].Type {
+			case restic.TreeBlob:
+				countTreePacks++
+			case restic.DataBlob:
+				countDataPacks++
+			}
+			countBlobs += len(blobs)
+			return nil
+		}))
+		return nil
+	}))
+
+	return countTreePacks, countDataPacks, countBlobs
 }
 
 func TestCopyIncremental(t *testing.T) {
@@ -142,7 +182,7 @@ func TestCopyToEmptyPassword(t *testing.T) {
 	defer cleanup()
 	env2, cleanup2 := withTestEnvironment(t)
 	defer cleanup2()
-	env2.gopts.password = ""
+	env2.gopts.Password = ""
 	env2.gopts.InsecureNoPassword = true
 
 	testSetupBackupData(t, env)

cmd/restic/cmd_debug.go

@@ -1,5 +1,4 @@
 //go:build debug
-// +build debug
 
 package main
 
@@ -18,27 +17,44 @@ import (
 
 	"github.com/klauspost/compress/zstd"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/crypto"
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/repository/index"
 	"github.com/restic/restic/internal/repository/pack"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/progress"
 )
 
-var cmdDebug = &cobra.Command{
+func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) {
-	Use:               "debug",
+	cmd.AddCommand(
-	Short:             "Debug commands",
+		newDebugCommand(globalOptions),
-	GroupID:           cmdGroupDefault,
+	)
-	DisableAutoGenTag: true,
 }
 
-var cmdDebugDump = &cobra.Command{
+func newDebugCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "dump [indexes|snapshots|all|packs]",
+	cmd := &cobra.Command{
-	Short: "Dump data structures",
+		Use:               "debug",
-	Long: `
+		Short:             "Debug commands",
+		GroupID:           cmdGroupDefault,
+		DisableAutoGenTag: true,
+	}
+	cmd.AddCommand(newDebugDumpCommand(globalOptions))
+	cmd.AddCommand(newDebugExamineCommand(globalOptions))
+	return cmd
+}
+
+func newDebugDumpCommand(globalOptions *global.Options) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "dump [indexes|snapshots|all|packs]",
+		Short: "Dump data structures",
+		Long: `
 The "dump" command dumps data structures from the repository as JSON objects. It
 is used for debugging purposes only.
 
@@ -51,10 +67,28 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugDump(cmd.Context(), globalOptions, args)
+			return runDebugDump(cmd.Context(), *globalOptions, args, globalOptions.Term)
-	},
+		},
+	}
+	return cmd
+}
+
+func newDebugExamineCommand(globalOptions *global.Options) *cobra.Command {
+	var opts DebugExamineOptions
+
+	cmd := &cobra.Command{
+		Use:               "examine pack-ID...",
+		Short:             "Examine a pack file",
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runDebugExamine(cmd.Context(), *globalOptions, opts, args, globalOptions.Term)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 type DebugExamineOptions struct {
@@ -64,16 +98,11 @@ type DebugExamineOptions struct {
 	ReuploadBlobs bool
 }
 
-var debugExamineOpts DebugExamineOptions
+func (opts *DebugExamineOptions) AddFlags(f *pflag.FlagSet) {
-
+	f.BoolVar(&opts.ExtractPack, "extract-pack", false, "write blobs to the current directory")
-func init() {
+	f.BoolVar(&opts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
-	cmdRoot.AddCommand(cmdDebug)
+	f.BoolVar(&opts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
-	cmdDebug.AddCommand(cmdDebugDump)
+	f.BoolVar(&opts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
-	cmdDebug.AddCommand(cmdDebugExamine)
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory")
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
-	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -87,12 +116,14 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 }
 
 func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
-	return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
+	return data.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *data.Snapshot, err error) error {
 		if err != nil {
 			return err
 		}
 
-		fmt.Fprintf(wr, "snapshot_id: %v\n", id)
+		if _, err := fmt.Fprintf(wr, "snapshot_id: %v\n", id); err != nil {
+			return err
+		}
 
 		return prettyPrintJSON(wr, snapshot)
 	})
@@ -113,13 +144,13 @@ type Blob struct {
 	Offset uint            `json:"offset"`
 }
 
-func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
+func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error {
 
 	var m sync.Mutex
 	return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
 		blobs, _, err := repo.ListPack(ctx, id, size)
 		if err != nil {
-			Warnf("error for pack %v: %v\n", id.Str(), err)
+			printer.E("error for pack %v: %v", id.Str(), err)
 			return nil
 		}
 
@@ -142,9 +173,9 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 	})
 }
 
-func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
+func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
-	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
-		Printf("index_id: %v\n", id)
+		printer.S("index_id: %v", id)
 		if err != nil {
 			return err
 		}
@@ -153,12 +184,14 @@ func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Wr
 	})
 }
 
-func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
+func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
+	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
+
 	if len(args) != 1 {
 		return errors.Fatal("type not specified")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
 		return err
 	}
@@ -168,20 +201,20 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 
 	switch tpe {
 	case "indexes":
-		return dumpIndexes(ctx, repo, globalOptions.stdout)
+		return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
 	case "snapshots":
-		return debugPrintSnapshots(ctx, repo, globalOptions.stdout)
+		return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
 	case "packs":
-		return printPacks(ctx, repo, globalOptions.stdout)
+		return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
 	case "all":
-		Printf("snapshots:\n")
+		printer.S("snapshots:")
-		err := debugPrintSnapshots(ctx, repo, globalOptions.stdout)
+		err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
 		if err != nil {
 			return err
 		}
 
-		Printf("\nindexes:\n")
+		printer.S("indexes:")
-		err = dumpIndexes(ctx, repo, globalOptions.stdout)
+		err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
 		if err != nil {
 			return err
 		}
@@ -192,20 +225,11 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 	}
 }
 
-var cmdDebugExamine = &cobra.Command{
+func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
-	Use:               "examine pack-ID...",
-	Short:             "Examine a pack file",
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args)
-	},
-}
-
-func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, bytewise bool) []byte {
 	if bytewise {
-		Printf("        trying to repair blob by finding a broken byte\n")
+		printer.S("        trying to repair blob by finding a broken byte")
 	} else {
-		Printf("        trying to repair blob with single bit flip\n")
+		printer.S("        trying to repair blob with single bit flip")
 	}
 
 	ch := make(chan int)
@@ -215,7 +239,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 	var found bool
 
 	workers := runtime.GOMAXPROCS(0)
-	Printf("         spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
+	printer.S("         spinning up %d worker functions", runtime.GOMAXPROCS(0))
 	for i := 0; i < workers; i++ {
 		wg.Go(func() error {
 			// make a local copy of the buffer
@@ -229,9 +253,9 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 				nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
 				plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
 				if err == nil {
-					Printf("\n")
+					printer.S("")
-					Printf("        blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
+					printer.S("        blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
-					Printf("        hash is %v\n", restic.Hash(plaintext))
+					printer.S("        hash is %v", restic.Hash(plaintext))
 					close(done)
 					found = true
 					fixed = plaintext
@@ -272,7 +296,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 			select {
 			case ch <- i:
 			case <-done:
-				Printf("     done after %v\n", time.Since(start))
+				printer.S("     done after %v", time.Since(start))
 				return nil
 			}
 
@@ -282,7 +306,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 				remaining := len(input) - i
 				eta := time.Duration(float64(remaining)/gps) * time.Second
 
-				Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
+				printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
 					i, len(input), float32(i)/float32(len(input))*100, gps, eta)
 				info = time.Now()
 			}
@@ -295,12 +319,12 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 	}
 
 	if !found {
-		Printf("\n        blob could not be repaired\n")
+		printer.S("\n        blob could not be repaired")
 	}
 	return fixed
 }
 
-func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
+func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
 	// strip signature at the end
 	l := len(buf)
 	nonce, ct := buf[:16], buf[16:l-16]
@@ -316,7 +340,7 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
 	return out
 }
 
-func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
+func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob, printer progress.Printer) error {
 	dec, err := zstd.NewReader(nil)
 	if err != nil {
 		panic(err)
@@ -328,17 +352,11 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
 		return err
 	}
 
-	wg, ctx := errgroup.WithContext(ctx)
+	err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
-
-	if opts.ReuploadBlobs {
-		repo.StartPackUploader(ctx, wg)
-	}
-
-	wg.Go(func() error {
 		for _, blob := range list {
-			Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
+			printer.S("      loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
 			if int(blob.Offset+blob.Length) > len(pack) {
-				Warnf("skipping truncated blob\n")
+				printer.E("skipping truncated blob")
 				continue
 			}
 			buf := pack[blob.Offset : blob.Offset+blob.Length]
@@ -349,16 +367,16 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
 			outputPrefix := ""
 			filePrefix := ""
 			if err != nil {
-				Warnf("error decrypting blob: %v\n", err)
+				printer.E("error decrypting blob: %v", err)
 				if opts.TryRepair || opts.RepairByte {
-					plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte)
+					plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
 				}
 				if plaintext != nil {
 					outputPrefix = "repaired "
 					filePrefix = "repaired-"
 				} else {
-					plaintext = decryptUnsigned(ctx, key, buf)
+					plaintext = decryptUnsigned(key, buf)
-					err = storePlainBlob(blob.ID, "damaged-", plaintext)
+					err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
 					if err != nil {
 						return err
 					}
@@ -369,7 +387,7 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
 			if blob.IsCompressed() {
 				decompressed, err := dec.DecodeAll(plaintext, nil)
 				if err != nil {
-					Printf("         failed to decompress blob %v\n", blob.ID)
+					printer.S("         failed to decompress blob %v", blob.ID)
 				}
 				if decompressed != nil {
 					plaintext = decompressed
@@ -379,37 +397,32 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
 			id := restic.Hash(plaintext)
 			var prefix string
 			if !id.Equal(blob.ID) {
-				Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
+				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
 				prefix = "wrong-hash-"
 			} else {
-				Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
+				printer.S("         successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
 				prefix = "correct-"
 			}
 			if opts.ExtractPack {
-				err = storePlainBlob(id, filePrefix+prefix, plaintext)
+				err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
 				if err != nil {
 					return err
 				}
 			}
 			if opts.ReuploadBlobs {
-				_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
+				_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
 				if err != nil {
 					return err
 				}
-				Printf("         uploaded %v %v\n", blob.Type, id)
+				printer.S("         uploaded %v %v", blob.Type, id)
 			}
 		}
-
-		if opts.ReuploadBlobs {
-			return repo.Flush(ctx)
-		}
 		return nil
 	})
-
+	return err
-	return wg.Wait()
 }
 
-func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
+func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
 	filename := fmt.Sprintf("%s%s.bin", prefix, id)
 	f, err := os.Create(filename)
 	if err != nil {
@@ -427,16 +440,18 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 		return err
 	}
 
-	Printf("decrypt of blob %v stored at %v\n", id, filename)
+	printer.S("decrypt of blob %v stored at %v", id, filename)
 	return nil
 }
 
-func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
+func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
+	printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
+
 	if opts.ExtractPack && gopts.NoLock {
 		return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive")
 	}
 
-	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock)
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
 		return err
 	}
@@ -448,7 +463,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamine
 		if err != nil {
 			id, err = restic.Find(ctx, repo, restic.PackFile, name)
 			if err != nil {
-				Warnf("error: %v\n", err)
+				printer.E("error: %v", err)
 				continue
 			}
 		}
@@ -459,16 +474,15 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamine
 		return errors.Fatal("no pack files to examine")
 	}
 
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, printer)
-	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
 	for _, id := range ids {
-		err := examinePack(ctx, opts, repo, id)
+		err := examinePack(ctx, opts, repo, id, printer)
 		if err != nil {
-			Warnf("error: %v\n", err)
+			printer.E("error: %v", err)
 		}
 		if err == context.Canceled {
 			break
@@ -477,24 +491,24 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamine
 	return nil
 }
 
-func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error {
+func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
-	Printf("examine %v\n", id)
+	printer.S("examine %v", id)
 
 	buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
 	// also process damaged pack files
 	if buf == nil {
 		return err
 	}
-	Printf("  file size is %v\n", len(buf))
+	printer.S("  file size is %v", len(buf))
 	gotID := restic.Hash(buf)
 	if !id.Equal(gotID) {
-		Printf("  wanted hash %v, got %v\n", id, gotID)
+		printer.S("  wanted hash %v, got %v", id, gotID)
 	} else {
-		Printf("  hash for file content matches\n")
+		printer.S("  hash for file content matches")
 	}
 
-	Printf("  ========================================\n")
+	printer.S("  ========================================")
-	Printf("  looking for info in the indexes\n")
+	printer.S("  looking for info in the indexes")
 
 	blobsLoaded := false
 	// examine all data the indexes have for the pack file
@@ -504,32 +518,32 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo
 			continue
 		}
 
-		checkPackSize(blobs, len(buf))
+		checkPackSize(blobs, len(buf), printer)
 
-		err = loadBlobs(ctx, opts, repo, id, blobs)
+		err = loadBlobs(ctx, opts, repo, id, blobs, printer)
 		if err != nil {
-			Warnf("error: %v\n", err)
+			printer.E("error: %v", err)
 		} else {
 			blobsLoaded = true
 		}
 	}
 
-	Printf("  ========================================\n")
+	printer.S("  ========================================")
-	Printf("  inspect the pack itself\n")
+	printer.S("  inspect the pack itself")
 
 	blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}
-	checkPackSize(blobs, len(buf))
+	checkPackSize(blobs, len(buf), printer)
 
 	if !blobsLoaded {
-		return loadBlobs(ctx, opts, repo, id, blobs)
+		return loadBlobs(ctx, opts, repo, id, blobs, printer)
 	}
 	return nil
 }
 
-func checkPackSize(blobs []restic.Blob, fileSize int) {
+func checkPackSize(blobs []restic.Blob, fileSize int, printer progress.Printer) {
 	// track current size and offset
 	var size, offset uint64
 
@@ -538,9 +552,9 @@ func checkPackSize(blobs []restic.Blob, fileSize int) {
 	})
 
 	for _, pb := range blobs {
-		Printf("      %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
+		printer.S("      %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
 		if offset != uint64(pb.Offset) {
-			Printf("      hole in file, want offset %v, got %v\n", offset, pb.Offset)
+			printer.S("      hole in file, want offset %v, got %v", offset, pb.Offset)
 		}
 		offset = uint64(pb.Offset + pb.Length)
 		size += uint64(pb.Length)
@@ -548,8 +562,8 @@ func checkPackSize(blobs []restic.Blob, fileSize int) {
 	size += uint64(pack.CalculateHeaderSize(blobs))
 
 	if uint64(fileSize) != size {
-		Printf("      file sizes do not match: computed %v, file size is %v\n", size, fileSize)
+		printer.S("      file sizes do not match: computed %v, file size is %v", size, fileSize)
 	} else {
-		Printf("      file sizes match\n")
+		printer.S("      file sizes match")
 	}
 }

cmd/restic/cmd_debug_disabled.go

@@ -0,0 +1,12 @@
+//go:build !debug
+
+package main
+
+import (
+	"github.com/restic/restic/internal/global"
+	"github.com/spf13/cobra"
+)
+
+func registerDebugCommand(_ *cobra.Command, _ *global.Options) {
+	// No commands to register in non-debug mode
+}

cmd/restic/cmd_diff.go

@@ -5,19 +5,24 @@ import (
 	"encoding/json"
 	"path"
 	"reflect"
-	"sort"
 
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdDiff = &cobra.Command{
+func newDiffCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "diff [flags] snapshotID snapshotID",
+	var opts DiffOptions
-	Short: "Show differences between two snapshots",
+
-	Long: `
+	cmd := &cobra.Command{
+		Use:   "diff [flags] snapshotID snapshotID",
+		Short: "Show differences between two snapshots",
+		Long: `
 The "diff" command shows differences from the first to the second snapshot. The
 first characters in each line display what has happened to a particular file or
 directory:
@@ -45,11 +50,15 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
+		GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		return runDiff(cmd.Context(), diffOptions, globalOptions, args)
+			return runDiff(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
-	},
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // DiffOptions collects all options for the diff command.
@@ -57,19 +66,14 @@ type DiffOptions struct {
 	ShowMetadata bool
 }
 
-var diffOptions DiffOptions
+func (opts *DiffOptions) AddFlags(f *pflag.FlagSet) {
-
+	f.BoolVar(&opts.ShowMetadata, "metadata", false, "print changes in metadata")
-func init() {
-	cmdRoot.AddCommand(cmdDiff)
-
-	f := cmdDiff.Flags()
-	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
-func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
+func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*data.Snapshot, string, error) {
-	sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
+	sn, subfolder, err := data.FindSnapshot(ctx, be, repo, desc)
 	if err != nil {
-		return nil, "", errors.Fatal(err.Error())
+		return nil, "", errors.Fatalf("%s", err)
 	}
 	return sn, subfolder, err
 }
@@ -79,6 +83,7 @@ type Comparer struct {
 	repo        restic.BlobLoader
 	opts        DiffOptions
 	printChange func(change *Change)
+	printError  func(string, ...interface{})
 }
 
 type Change struct {
@@ -102,15 +107,15 @@ type DiffStat struct {
 }
 
 // Add adds stats information for node to s.
-func (s *DiffStat) Add(node *restic.Node) {
+func (s *DiffStat) Add(node *data.Node) {
 	if node == nil {
 		return
 	}
 
 	switch node.Type {
-	case "file":
+	case data.NodeTypeFile:
 		s.Files++
-	case "dir":
+	case data.NodeTypeDir:
 		s.Dirs++
 	default:
 		s.Others++
@@ -118,13 +123,13 @@ func (s *DiffStat) Add(node *restic.Node) {
 }
 
 // addBlobs adds the blobs of node to s.
-func addBlobs(bs restic.BlobSet, node *restic.Node) {
+func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
 	if node == nil {
 		return
 	}
 
 	switch node.Type {
-	case "file":
+	case data.NodeTypeFile:
 		for _, blob := range node.Content {
 			h := restic.BlobHandle{
 				ID:   blob,
@@ -132,7 +137,7 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 			}
 			bs.Insert(h)
 		}
-	case "dir":
+	case data.NodeTypeDir:
 		h := restic.BlobHandle{
 			ID:   *node.Subtree,
 			Type: restic.TreeBlob,
@@ -142,18 +147,18 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 }
 
 type DiffStatsContainer struct {
-	MessageType                          string         `json:"message_type"` // "statistics"
+	MessageType                          string                   `json:"message_type"` // "statistics"
-	SourceSnapshot                       string         `json:"source_snapshot"`
+	SourceSnapshot                       string                   `json:"source_snapshot"`
-	TargetSnapshot                       string         `json:"target_snapshot"`
+	TargetSnapshot                       string                   `json:"target_snapshot"`
-	ChangedFiles                         int            `json:"changed_files"`
+	ChangedFiles                         int                      `json:"changed_files"`
-	Added                                DiffStat       `json:"added"`
+	Added                                DiffStat                 `json:"added"`
-	Removed                              DiffStat       `json:"removed"`
+	Removed                              DiffStat                 `json:"removed"`
-	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
+	BlobsBefore, BlobsAfter, BlobsCommon restic.AssociatedBlobSet `json:"-"`
 }
 
 // updateBlobs updates the blob counters in the stats struct.
-func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
+func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *DiffStat, printError func(string, ...interface{})) {
-	for h := range blobs {
+	for h := range blobs.Keys() {
 		switch h.Type {
 		case restic.DataBlob:
 			stats.DataBlobs++
@@ -163,7 +168,7 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
 
 		size, found := repo.LookupBlobSize(h.Type, h.ID)
 		if !found {
-			Warnf("unable to find blob size for %v\n", h)
+			printError("unable to find blob size for %v", h)
 			continue
 		}
 
@@ -171,30 +176,33 @@ func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
 	}
 }
 
-func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
+func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.AssociatedBlobSet, prefix string, id restic.ID) error {
 	debug.Log("print %v tree %v", mode, id)
-	tree, err := restic.LoadTree(ctx, c.repo, id)
+	tree, err := data.LoadTree(ctx, c.repo, id)
 	if err != nil {
 		return err
 	}
 
-	for _, node := range tree.Nodes {
+	for item := range tree {
+		if item.Error != nil {
+			return item.Error
+		}
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
-
+		node := item.Node
 		name := path.Join(prefix, node.Name)
-		if node.Type == "dir" {
+		if node.Type == data.NodeTypeDir {
 			name += "/"
 		}
 		c.printChange(NewChange(name, mode))
 		stats.Add(node)
 		addBlobs(blobs, node)
 
-		if node.Type == "dir" {
+		if node.Type == data.NodeTypeDir {
 			err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
 			if err != nil && err != context.Canceled {
-				Warnf("error: %v\n", err)
+				c.printError("error: %v", err)
 			}
 		}
 	}
@@ -202,24 +210,28 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 	return ctx.Err()
 }
 
-func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
+func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSet, id restic.ID) error {
 	debug.Log("print tree %v", id)
-	tree, err := restic.LoadTree(ctx, c.repo, id)
+	tree, err := data.LoadTree(ctx, c.repo, id)
 	if err != nil {
 		return err
 	}
 
-	for _, node := range tree.Nodes {
+	for item := range tree {
+		if item.Error != nil {
+			return item.Error
+		}
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
 
+		node := item.Node
 		addBlobs(blobs, node)
 
-		if node.Type == "dir" {
+		if node.Type == data.NodeTypeDir {
 			err := c.collectDir(ctx, blobs, *node.Subtree)
 			if err != nil && err != context.Canceled {
-				Warnf("error: %v\n", err)
+				c.printError("error: %v", err)
 			}
 		}
 	}
@@ -227,56 +239,41 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest
 	return ctx.Err()
 }
 
-func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) {
-	names := make(map[string]struct{})
-	tree1Nodes = make(map[string]*restic.Node)
-	for _, node := range tree1.Nodes {
-		tree1Nodes[node.Name] = node
-		names[node.Name] = struct{}{}
-	}
-
-	tree2Nodes = make(map[string]*restic.Node)
-	for _, node := range tree2.Nodes {
-		tree2Nodes[node.Name] = node
-		names[node.Name] = struct{}{}
-	}
-
-	uniqueNames = make([]string, 0, len(names))
-	for name := range names {
-		uniqueNames = append(uniqueNames, name)
-	}
-
-	sort.Strings(uniqueNames)
-	return tree1Nodes, tree2Nodes, uniqueNames
-}
-
 func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
-	tree1, err := restic.LoadTree(ctx, c.repo, id1)
+	tree1, err := data.LoadTree(ctx, c.repo, id1)
 	if err != nil {
 		return err
 	}
 
-	tree2, err := restic.LoadTree(ctx, c.repo, id2)
+	tree2, err := data.LoadTree(ctx, c.repo, id2)
 	if err != nil {
 		return err
 	}
 
-	tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2)
+	for dt := range data.DualTreeIterator(tree1, tree2) {
-
+		if dt.Error != nil {
-	for _, name := range names {
+			return dt.Error
+		}
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
 
-		node1, t1 := tree1Nodes[name]
+		node1 := dt.Tree1
-		node2, t2 := tree2Nodes[name]
+		node2 := dt.Tree2
+
+		var name string
+		if node1 != nil {
+			name = node1.Name
+		} else {
+			name = node2.Name
+		}
 
 		addBlobs(stats.BlobsBefore, node1)
 		addBlobs(stats.BlobsAfter, node2)
 
 		switch {
-		case t1 && t2:
+		case node1 != nil && node2 != nil:
 			name := path.Join(prefix, name)
 			mod := ""
 
@@ -284,12 +281,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 				mod += "T"
 			}
 
-			if node2.Type == "dir" {
+			if node2.Type == data.NodeTypeDir {
 				name += "/"
 			}
 
-			if node1.Type == "file" &&
+			if node1.Type == data.NodeTypeFile &&
-				node2.Type == "file" &&
+				node2.Type == data.NodeTypeFile &&
 				!reflect.DeepEqual(node1.Content, node2.Content) {
 				mod += "M"
 				stats.ChangedFiles++
@@ -311,7 +308,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 				c.printChange(NewChange(name, mod))
 			}
 
-			if node1.Type == "dir" && node2.Type == "dir" {
+			if node1.Type == data.NodeTypeDir && node2.Type == data.NodeTypeDir {
 				var err error
 				if (*node1.Subtree).Equal(*node2.Subtree) {
 					err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree)
@@ -319,35 +316,35 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 					err = c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree)
 				}
 				if err != nil && err != context.Canceled {
-					Warnf("error: %v\n", err)
+					c.printError("error: %v", err)
 				}
 			}
-		case t1 && !t2:
+		case node1 != nil && node2 == nil:
 			prefix := path.Join(prefix, name)
-			if node1.Type == "dir" {
+			if node1.Type == data.NodeTypeDir {
 				prefix += "/"
 			}
 			c.printChange(NewChange(prefix, "-"))
 			stats.Removed.Add(node1)
 
-			if node1.Type == "dir" {
+			if node1.Type == data.NodeTypeDir {
 				err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
 				if err != nil && err != context.Canceled {
-					Warnf("error: %v\n", err)
+					c.printError("error: %v", err)
 				}
 			}
-		case !t1 && t2:
+		case node1 == nil && node2 != nil:
 			prefix := path.Join(prefix, name)
-			if node2.Type == "dir" {
+			if node2.Type == data.NodeTypeDir {
 				prefix += "/"
 			}
 			c.printChange(NewChange(prefix, "+"))
 			stats.Added.Add(node2)
 
-			if node2.Type == "dir" {
+			if node2.Type == data.NodeTypeDir {
 				err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
 				if err != nil && err != context.Canceled {
-					Warnf("error: %v\n", err)
+					c.printError("error: %v", err)
 				}
 			}
 		}
@@ -356,12 +353,14 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 	return ctx.Err()
 }
 
-func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
+func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args []string, term ui.Terminal) error {
 	if len(args) != 2 {
 		return errors.Fatalf("specify two snapshot IDs")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
+
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
 		return err
 	}
@@ -383,10 +382,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	}
 
 	if !gopts.JSON {
-		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+		printer.P("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
 	}
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, printer); err != nil {
-	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
@@ -398,30 +396,31 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
 	}
 
-	sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
+	sn1.Tree, err = data.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
 	if err != nil {
 		return err
 	}
 
-	sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
+	sn2.Tree, err = data.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
 	if err != nil {
 		return err
 	}
 
 	c := &Comparer{
-		repo: repo,
+		repo:       repo,
-		opts: opts,
+		opts:       opts,
+		printError: printer.E,
 		printChange: func(change *Change) {
-			Printf("%-5s%v\n", change.Modifier, change.Path)
+			printer.S("%-5s%v", change.Modifier, change.Path)
 		},
 	}
 
 	if gopts.JSON {
-		enc := json.NewEncoder(globalOptions.stdout)
+		enc := json.NewEncoder(gopts.Term.OutputWriter())
 		c.printChange = func(change *Change) {
 			err := enc.Encode(change)
 			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
+				printer.E("JSON encode failed: %v", err)
 			}
 		}
 	}
@@ -434,9 +433,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		MessageType:    "statistics",
 		SourceSnapshot: args[0],
 		TargetSnapshot: args[1],
-		BlobsBefore:    restic.NewBlobSet(),
+		BlobsBefore:    repo.NewAssociatedBlobSet(),
-		BlobsAfter:     restic.NewBlobSet(),
+		BlobsAfter:     repo.NewAssociatedBlobSet(),
-		BlobsCommon:    restic.NewBlobSet(),
+		BlobsCommon:    repo.NewAssociatedBlobSet(),
 	}
 	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
 	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
@@ -447,23 +446,23 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	}
 
 	both := stats.BlobsBefore.Intersect(stats.BlobsAfter)
-	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
+	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed, printer.E)
-	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
+	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added, printer.E)
 
 	if gopts.JSON {
-		err := json.NewEncoder(globalOptions.stdout).Encode(stats)
+		err := json.NewEncoder(gopts.Term.OutputWriter()).Encode(stats)
 		if err != nil {
-			Warnf("JSON encode failed: %v\n", err)
+			printer.E("JSON encode failed: %v", err)
 		}
 	} else {
-		Printf("\n")
+		printer.S("")
-		Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
+		printer.S("Files:       %5d new, %5d removed, %5d changed", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
-		Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
+		printer.S("Dirs:        %5d new, %5d removed", stats.Added.Dirs, stats.Removed.Dirs)
-		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
+		printer.S("Others:      %5d new, %5d removed", stats.Added.Others, stats.Removed.Others)
-		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
+		printer.S("Data Blobs:  %5d new, %5d removed", stats.Added.DataBlobs, stats.Removed.DataBlobs)
-		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
+		printer.S("Tree Blobs:  %5d new, %5d removed", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
-		Printf("  Added:   %-5s\n", ui.FormatBytes(uint64(stats.Added.Bytes)))
+		printer.S("  Added:   %-5s", ui.FormatBytes(stats.Added.Bytes))
-		Printf("  Removed: %-5s\n", ui.FormatBytes(uint64(stats.Removed.Bytes)))
+		printer.S("  Removed: %-5s", ui.FormatBytes(stats.Removed.Bytes))
 	}
 
 	return nil

cmd/restic/cmd_diff_integration_test.go

@@ -11,15 +11,16 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/global"
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) {
+func testRunDiffOutput(t testing.TB, gopts global.Options, firstSnapshotID string, secondSnapshotID string) (string, error) {
-	buf, err := withCaptureStdout(func() error {
+	buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
 		opts := DiffOptions{
 			ShowMetadata: false,
 		}
-		return runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
+		return runDiff(ctx, opts, gopts, []string{firstSnapshotID, secondSnapshotID}, gopts.Term)
 	})
 	return buf.String(), err
 }
@@ -123,10 +124,10 @@ func TestDiff(t *testing.T) {
 
 	// quiet suppresses the diff output except for the summary
 	env.gopts.Quiet = false
-	_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
+	_, err := testRunDiffOutput(t, env.gopts, "", secondSnapshotID)
 	rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
 
-	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	out, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
 	rtest.OK(t, err)
 
 	for _, pattern := range diffOutputRegexPatterns {
@@ -137,7 +138,7 @@ func TestDiff(t *testing.T) {
 
 	// check quiet output
 	env.gopts.Quiet = true
-	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	outQuiet, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
 	rtest.OK(t, err)
 
 	rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
@@ -154,7 +155,7 @@ func TestDiffJSON(t *testing.T) {
 	// quiet suppresses the diff output except for the summary
 	env.gopts.Quiet = false
 	env.gopts.JSON = true
-	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	out, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
 	rtest.OK(t, err)
 
 	var stat DiffStatsContainer
@@ -181,7 +182,7 @@ func TestDiffJSON(t *testing.T) {
 
 	// check quiet output
 	env.gopts.Quiet = true
-	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	outQuiet, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
 	rtest.OK(t, err)
 
 	stat = DiffStatsContainer{}

cmd/restic/cmd_dump.go

@@ -7,18 +7,24 @@ import (
 	"path"
 	"path/filepath"
 
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/dump"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdDump = &cobra.Command{
+func newDumpCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "dump [flags] snapshotID file",
+	var opts DumpOptions
-	Short: "Print a backed-up file to stdout",
+	cmd := &cobra.Command{
-	Long: `
+		Use:   "dump [flags] snapshotID file",
+		Short: "Print a backed-up file to stdout",
+		Long: `
 The "dump" command extracts files from a snapshot from the repository. If a
 single file is selected, it prints its contents to stdout. Folders are output
 as a tar (default) or zip file containing the contents of the specified folder.
@@ -40,29 +46,29 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
+		GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		return runDump(cmd.Context(), dumpOptions, globalOptions, args)
+			finalizeSnapshotFilter(&opts.SnapshotFilter)
-	},
+			return runDump(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // DumpOptions collects all options for the dump command.
 type DumpOptions struct {
-	restic.SnapshotFilter
+	data.SnapshotFilter
 	Archive string
 	Target  string
 }
 
-var dumpOptions DumpOptions
+func (opts *DumpOptions) AddFlags(f *pflag.FlagSet) {
-
+	initSingleSnapshotFilter(f, &opts.SnapshotFilter)
-func init() {
+	f.StringVarP(&opts.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
-	cmdRoot.AddCommand(cmdDump)
+	f.StringVarP(&opts.Target, "target", "t", "", "write the output to target `path`")
-
-	flags := cmdDump.Flags()
-	initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
-	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
-	flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`")
 }
 
 func splitPath(p string) []string {
@@ -74,7 +80,7 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
+func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
@@ -86,35 +92,38 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade
 
 	item := filepath.Join(prefix, pathComponents[0])
 	l := len(pathComponents)
-	for _, node := range tree.Nodes {
+	for it := range tree {
+		if it.Error != nil {
+			return it.Error
+		}
 		if ctx.Err() != nil {
 			return ctx.Err()
 		}
-
+		node := it.Node
 		// If dumping something in the highest level it will just take the
 		// first item it finds and dump that according to the switch case below.
 		if node.Name == pathComponents[0] {
 			switch {
-			case l == 1 && dump.IsFile(node):
+			case l == 1 && node.Type == data.NodeTypeFile:
 				return d.WriteNode(ctx, node)
-			case l > 1 && dump.IsDir(node):
+			case l > 1 && node.Type == data.NodeTypeDir:
-				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
+				subtree, err := data.LoadTree(ctx, repo, *node.Subtree)
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
 				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
-			case dump.IsDir(node):
+			case node.Type == data.NodeTypeDir:
 				if err := canWriteArchiveFunc(); err != nil {
 					return err
 				}
-				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
+				subtree, err := data.LoadTree(ctx, repo, *node.Subtree)
 				if err != nil {
 					return err
 				}
 				return d.DumpTree(ctx, subtree, item)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
-			case !dump.IsFile(node):
+			case node.Type != data.NodeTypeFile:
 				return fmt.Errorf("%q should be a file, but is a %q", item, node.Type)
 			}
 		}
@@ -122,11 +131,13 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade
 	return fmt.Errorf("path %q not found in snapshot", item)
 }
 
-func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
+func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args []string, term ui.Terminal) error {
 	if len(args) != 2 {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 
+	printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
+
 	switch opts.Archive {
 	case "tar", "zip":
 	default:
@@ -140,39 +151,34 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 
 	splittedPath := splitPath(path.Clean(pathToPrint))
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
 		return err
 	}
 	defer unlock()
 
-	sn, subfolder, err := (&restic.SnapshotFilter{
+	sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString)
-		Hosts: opts.Hosts,
-		Paths: opts.Paths,
-		Tags:  opts.Tags,
-	}).FindLatest(ctx, repo, repo, snapshotIDString)
 	if err != nil {
 		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
 
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	err = repo.LoadIndex(ctx, printer)
-	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
-	sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
+	sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
 	if err != nil {
 		return err
 	}
 
-	tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
+	tree, err := data.LoadTree(ctx, repo, *sn.Tree)
 	if err != nil {
 		return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	outputFileWriter := os.Stdout
+	outputFileWriter := term.OutputRaw()
-	canWriteArchiveFunc := checkStdoutArchive
+	canWriteArchiveFunc := checkStdoutArchive(term)
 
 	if opts.Target != "" {
 		file, err := os.Create(opts.Target)
@@ -196,9 +202,9 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 	return nil
 }
 
-func checkStdoutArchive() error {
+func checkStdoutArchive(term ui.Terminal) func() error {
-	if stdoutIsTerminal() {
+	if term.OutputIsTerminal() {
-		return fmt.Errorf("stdout is the terminal, please redirect output")
+		return func() error { return fmt.Errorf("stdout is the terminal, please redirect output") }
 	}
-	return nil
+	return func() error { return nil }
 }

cmd/restic/cmd_features.go

@@ -1,19 +1,19 @@
 package main
 
 import (
-	"fmt"
-
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/feature"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/ui/table"
 
 	"github.com/spf13/cobra"
 )
 
-var featuresCmd = &cobra.Command{
+func newFeaturesCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "features",
+	cmd := &cobra.Command{
-	Short: "Print list of feature flags",
+		Use:   "features",
-	Long: `
+		Short: "Print list of feature flags",
+		Long: `
 The "features" command prints a list of supported feature flags.
 
 To pass feature flags to restic, set the RESTIC_FEATURES environment variable
@@ -31,29 +31,28 @@ EXIT STATUS
 Exit status is 0 if the command was successful.
 Exit status is 1 if there was any error.
 `,
-	GroupID:           cmdGroupAdvanced,
+		GroupID:           cmdGroupAdvanced,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(_ *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
-		if len(args) != 0 {
+			if len(args) != 0 {
-			return errors.Fatal("the feature command expects no arguments")
+				return errors.Fatal("the feature command expects no arguments")
-		}
+			}
 
-		fmt.Printf("All Feature Flags:\n")
+			globalOptions.Term.Print("All Feature Flags:\n")
-		flags := feature.Flag.List()
+			flags := feature.Flag.List()
 
-		tab := table.New()
+			tab := table.New()
-		tab.AddColumn("Name", "{{ .Name }}")
+			tab.AddColumn("Name", "{{ .Name }}")
-		tab.AddColumn("Type", "{{ .Type }}")
+			tab.AddColumn("Type", "{{ .Type }}")
-		tab.AddColumn("Default", "{{ .Default }}")
+			tab.AddColumn("Default", "{{ .Default }}")
-		tab.AddColumn("Description", "{{ .Description }}")
+			tab.AddColumn("Description", "{{ .Description }}")
 
-		for _, flag := range flags {
+			for _, flag := range flags {
-			tab.AddRow(flag)
+				tab.AddRow(flag)
-		}
+			}
-		return tab.Write(globalOptions.stdout)
+			return tab.Write(globalOptions.Term.OutputWriter())
-	},
+		},
-}
+	}
-
+
-func init() {
+	return cmd
-	cmdRoot.AddCommand(featuresCmd)
 }

cmd/restic/cmd_find.go

@@ -3,27 +3,38 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"io"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 
+	"github.com/restic/restic/internal/data"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
+	"github.com/restic/restic/internal/global"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/walker"
 )
 
-var cmdFind = &cobra.Command{
+func newFindCommand(globalOptions *global.Options) *cobra.Command {
-	Use:   "find [flags] PATTERN...",
+	var opts FindOptions
-	Short: "Find a file, a directory or restic IDs",
+
-	Long: `
+	cmd := &cobra.Command{
+		Use:   "find [flags] PATTERN...",
+		Short: "Find a file, a directory or restic IDs",
+		Long: `
 The "find" command searches for files or directories in snapshots stored in the
 repo.
-It can also be used to search for restic blobs or trees for troubleshooting.`,
+It can also be used to search for restic blobs or trees for troubleshooting.
-	Example: `restic find config.json
+The default sort option for the snapshots is youngest to oldest. To sort the
+output from oldest to youngest specify --reverse.`,
+		Example: `restic find config.json
 restic find --json "*.yml" "*.json"
 restic find --json --blob 420f620f b46ebe8a ddd38656
 restic find --show-pack-id --blob 420f620f
@@ -39,11 +50,16 @@ Exit status is 10 if the repository does not exist.
 Exit status is 11 if the repository is already locked.
 Exit status is 12 if the password is incorrect.
 `,
-	GroupID:           cmdGroupDefault,
+		GroupID:           cmdGroupDefault,
-	DisableAutoGenTag: true,
+		DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		return runFind(cmd.Context(), findOptions, globalOptions, args)
+			finalizeSnapshotFilter(&opts.SnapshotFilter)
-	},
+			return runFind(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
+		},
+	}
+
+	opts.AddFlags(cmd.Flags())
+	return cmd
 }
 
 // FindOptions bundles all options for the find command.
@@ -56,27 +72,24 @@ type FindOptions struct {
 	CaseInsensitive    bool
 	ListLong           bool
 	HumanReadable      bool
-	restic.SnapshotFilter
+	Reverse            bool
+	data.SnapshotFilter
 }
 
-var findOptions FindOptions
+func (opts *FindOptions) AddFlags(f *pflag.FlagSet) {
+	f.StringVarP(&opts.Oldest, "oldest", "O", "", "oldest modification date/time")
+	f.StringVarP(&opts.Newest, "newest", "N", "", "newest modification date/time")
+	f.StringArrayVarP(&opts.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.BoolVar(&opts.BlobID, "blob", false, "pattern is a blob-ID")
+	f.BoolVar(&opts.TreeID, "tree", false, "pattern is a tree-ID")
+	f.BoolVar(&opts.PackID, "pack", false, "pattern is a pack-ID")
+	f.BoolVar(&opts.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob or --tree)")
+	f.BoolVarP(&opts.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
+	f.BoolVarP(&opts.Reverse, "reverse", "R", false, "reverse sort order oldest to newest")
+	f.BoolVarP(&opts.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+	f.BoolVar(&opts.HumanReadable, "human-readable", false, "print sizes in human readable format")
 
-func init() {
+	initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
-	cmdRoot.AddCommand(cmdFind)
-
-	f := cmdFind.Flags()
-	f.StringVarP(&findOptions.Oldest, "oldest", "O", "", "oldest modification date/time")
-	f.StringVarP(&findOptions.Newest, "newest", "N", "", "newest modification date/time")
-	f.StringArrayVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
-	f.BoolVar(&findOptions.BlobID, "blob", false, "pattern is a blob-ID")
-	f.BoolVar(&findOptions.TreeID, "tree", false, "pattern is a tree-ID")
-	f.BoolVar(&findOptions.PackID, "pack", false, "pattern is a pack-ID")
-	f.BoolVar(&findOptions.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob or --tree)")
-	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
-	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	f.BoolVar(&findOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
-
-	initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true)
 }
 
 type findPattern struct {
@@ -114,13 +127,19 @@ type statefulOutput struct {
 	HumanReadable bool
 	JSON          bool
 	inuse         bool
-	newsn         *restic.Snapshot
+	newsn         *data.Snapshot
-	oldsn         *restic.Snapshot
+	oldsn         *data.Snapshot
 	hits          int
+	printer       interface {
+		S(string, ...interface{})
+		P(string, ...interface{})
+		E(string, ...interface{})
+	}
+	stdout io.Writer
 }
 
-func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
+func (s *statefulOutput) PrintPatternJSON(path string, node *data.Node) {
-	type findNode restic.Node
+	type findNode data.Node
 	b, err := json.Marshal(struct {
 		// Add these attributes
 		Path        string `json:"path,omitempty"`
@@ -141,40 +160,40 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
 		findNode:    (*findNode)(node),
 	})
 	if err != nil {
-		Warnf("Marshall failed: %v\n", err)
+		s.printer.E("Marshall failed: %v", err)
 		return
 	}
 	if !s.inuse {
-		Printf("[")
+		_, _ = s.stdout.Write([]byte("["))
 		s.inuse = true
 	}
 	if s.newsn != s.oldsn {
 		if s.oldsn != nil {
-			Printf("],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID())
+			_, _ = fmt.Fprintf(s.stdout, "],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID())
 		}
-		Printf(`{"matches":[`)
+		_, _ = s.stdout.Write([]byte(`{"matches":[`))
 		s.oldsn = s.newsn
 		s.hits = 0
 	}
 	if s.hits > 0 {
-		Printf(",")
+		_, _ = s.stdout.Write([]byte(","))
 	}
-	Print(string(b))
+	_, _ = s.stdout.Write(b)
 	s.hits++
 }
 
-func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) {
+func (s *statefulOutput) PrintPatternNormal(path string, node *data.Node) {
 	if s.newsn != s.oldsn {
 		if s.oldsn != nil {
-			Verbosef("\n")
+			s.printer.P("")
 		}
 		s.oldsn = s.newsn
-		Verbosef("Found matching entries in snapshot %s from %s\n", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(TimeFormat))
+		s.printer.P("Found matching entries in snapshot %s from %s", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(global.TimeFormat))
 	}
-	Println(formatNode(path, node, s.ListLong, s.HumanReadable))
+	s.printer.S(formatNode(path, node, s.ListLong, s.HumanReadable))
 }
 
-func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
+func (s *statefulOutput) PrintPattern(path string, node *data.Node) {
 	if s.JSON {
 		s.PrintPatternJSON(path, node)
 	} else {
@@ -182,7 +201,7 @@ func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
 	}
 }
 
-func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *data.Snapshot) {
 	b, err := json.Marshal(struct {
 		// Add these attributes
 		ObjectType string    `json:"object_type"`
@@ -200,32 +219,32 @@ func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *
 		Time:       sn.Time,
 	})
 	if err != nil {
-		Warnf("Marshall failed: %v\n", err)
+		s.printer.E("Marshall failed: %v", err)
 		return
 	}
 	if !s.inuse {
-		Printf("[")
+		_, _ = s.stdout.Write([]byte("["))
 		s.inuse = true
 	}
 	if s.hits > 0 {
-		Printf(",")
+		_, _ = s.stdout.Write([]byte(","))
 	}
-	Print(string(b))
+	_, _ = s.stdout.Write(b)
 	s.hits++
 }
 
-func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *data.Snapshot) {
-	Printf("Found %s %s\n", kind, id)
+	s.printer.S("Found %s %s", kind, id)
 	if kind == "blob" {
-		Printf(" ... in file %s\n", nodepath)
+		s.printer.S(" ... in file %s", nodepath)
-		Printf("     (tree %s)\n", treeID)
+		s.printer.S("     (tree %s)", treeID)
 	} else {
-		Printf(" ... path %s\n", nodepath)
+		s.printer.S(" ... path %s", nodepath)
 	}
-	Printf(" ... in snapshot %s (%s)\n", sn.ID().Str(), sn.Time.Local().Format(TimeFormat))
+	s.printer.S(" ... in snapshot %s (%s)", sn.ID().Str(), sn.Time.Local().Format(global.TimeFormat))
 }
 
-func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *data.Snapshot) {
 	if s.JSON {
 		s.PrintObjectJSON(kind, id, nodepath, treeID, sn)
 	} else {
@@ -237,12 +256,12 @@ func (s *statefulOutput) Finish() {
 	if s.JSON {
 		// do some finishing up
 		if s.oldsn != nil {
-			Printf("],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID())
+			_, _ = fmt.Fprintf(s.stdout, "],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID())
 		}
 		if s.inuse {
-			Printf("]\n")
+			_, _ = s.stdout.Write([]byte("]\n"))
 		} else {
-			Printf("[]\n")
+			_, _ = s.stdout.Write([]byte("[]\n"))
 		}
 		return
 	}
@@ -256,9 +275,14 @@ type Finder struct {
 	blobIDs    map[string]struct{}
 	treeIDs    map[string]struct{}
 	itemsFound int
+	printer    interface {
+		S(string, ...interface{})
+		P(string, ...interface{})
+		E(string, ...interface{})
+	}
 }
 
-func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
+func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error {
 	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
 
 	if sn.Tree == nil {
@@ -266,11 +290,12 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
+			f.printer.S("Unable to load tree %s", parentTreeID)
+			f.printer.S(" ... which belongs to snapshot %s", sn.ID())
 
 			return walker.ErrSkipNode
 		}
@@ -298,7 +323,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		}
 
 		var errIfNoMatch error
-		if node.Type == "dir" {
+		if node.Type == data.NodeTypeDir {
 			var childMayMatch bool
 			for _, pat := range f.pat.pattern {
 				mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
@@ -336,7 +361,27 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}})
 }
 
-func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
+func (f *Finder) findTree(treeID restic.ID, nodepath string) error {
+	found := false
+	if _, ok := f.treeIDs[treeID.String()]; ok {
+		found = true
+	} else if _, ok := f.treeIDs[treeID.Str()]; ok {
+		found = true
+	}
+	if found {
+		f.out.PrintObject("tree", treeID.String(), nodepath, "", f.out.newsn)
+		f.itemsFound++
+		// Terminate if we have found all trees (and we are not
+		// looking for blobs)
+		if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
+			// Return an error to terminate the Walk
+			return errors.New("OK")
+		}
+	}
+	return nil
+}
+
+func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error {
 	debug.Log("searching IDs in snapshot %s", sn.ID())
 
 	if sn.Tree == nil {
@@ -344,40 +389,32 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
-			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
+			f.printer.S("Unable to load tree %s", parentTreeID)
+			f.printer.S(" ... which belongs to snapshot %s", sn.ID())
 
 			return walker.ErrSkipNode
 		}
 
 		if node == nil {
+			if nodepath == "/" {
+				if err := f.findTree(parentTreeID, "/"); err != nil {
+					return err
+				}
+			}
 			return nil
 		}
 
 		if node.Type == "dir" && f.treeIDs != nil {
-			treeID := node.Subtree
+			if err := f.findTree(*node.Subtree, nodepath); err != nil {
-			found := false
+				return err
-			if _, ok := f.treeIDs[treeID.Str()]; ok {
-				found = true
-			} else if _, ok := f.treeIDs[treeID.String()]; ok {
-				found = true
-			}
-			if found {
-				f.out.PrintObject("tree", treeID.String(), nodepath, "", sn)
-				f.itemsFound++
-				// Terminate if we have found all trees (and we are not
-				// looking for blobs)
-				if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
-					// Return an error to terminate the Walk
-					return errors.New("OK")
-				}
 			}
 		}
 
-		if node.Type == "file" && f.blobIDs != nil {
+		if node.Type == data.NodeTypeFile && f.blobIDs != nil {
 			for _, id := range node.Content {
 				if ctx.Err() != nil {
 					return ctx.Err()
@@ -413,6 +450,9 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 	if f.blobIDs == nil {
 		f.blobIDs = make(map[string]struct{})
 	}
+	if f.treeIDs == nil {
+		f.treeIDs = make(map[string]struct{})
+	}
 
 	debug.Log("Looking for packs...")
 	err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
@@ -433,7 +473,14 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 			return err
 		}
 		for _, b := range blobs {
-			f.blobIDs[b.ID.String()] = struct{}{}
+			switch b.Type {
+			case restic.DataBlob:
+				f.blobIDs[b.ID.String()] = struct{}{}
+			case restic.TreeBlob:
+				f.treeIDs[b.ID.String()] = struct{}{}
+			default:
+				panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String()))
+			}
 		}
 		// Stop searching when all packs have been found
 		if len(packIDs) == 0 {
@@ -506,7 +553,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 		for h := range indexPackIDs {
 			list = append(list, h)
 		}
-		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
+		f.printer.E("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
 	}
 	return packIDs, nil
 }
@@ -514,19 +561,20 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 func (f *Finder) findObjectPack(id string, t restic.BlobType) {
 	rid, err := restic.ParseID(id)
 	if err != nil {
-		Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err)
+		f.printer.S("Note: cannot find pack for object '%s', unable to parse ID: %v", id, err)
 		return
 	}
 
 	blobs := f.repo.LookupBlob(t, rid)
 	if len(blobs) == 0 {
-		Printf("Object %s not found in the index\n", rid.Str())
+		f.printer.S("Object %s with type %s not found in the index", t.String(), rid.Str())
 		return
 	}
 
 	for _, b := range blobs {
 		if b.ID.Equal(rid) {
-			Printf("Object belongs to pack %s\n ... Pack %s: %s\n", b.PackID, b.PackID.Str(), b.String())
+			f.printer.S("Object belongs to pack %s", b.PackID)
+			f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String())
 			break
 		}
 	}
@@ -542,11 +590,13 @@ func (f *Finder) findObjectsPacks() {
 	}
 }
 
-func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
+func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args []string, term ui.Terminal) error {
 	if len(args) == 0 {
 		return errors.Fatal("wrong number of arguments")
 	}
 
+	printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
+
 	var err error
 	pat := findPattern{pattern: args}
 	if opts.CaseInsensitive {
@@ -568,6 +618,10 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		}
 	}
 
+	if !pat.newest.IsZero() && !pat.oldest.IsZero() && pat.oldest.After(pat.newest) {
+		return errors.Fatal("--oldest must specify a time before --newest")
+	}
+
 	// Check at most only one kind of IDs is provided: currently we
 	// can't mix types
 	if (opts.BlobID && opts.TreeID) ||
@@ -576,7 +630,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		return errors.Fatal("cannot have several ID types")
 	}
 
-	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
 	if err != nil {
 		return err
 	}
@@ -586,15 +640,15 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	if err != nil {
 		return err
 	}
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	if err = repo.LoadIndex(ctx, printer); err != nil {
-	if err = repo.LoadIndex(ctx, bar); err != nil {
 		return err
 	}
 
 	f := &Finder{
-		repo: repo,
+		repo:    repo,
-		pat:  pat,
+		pat:     pat,
-		out:  statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
+		out:     statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON, printer: printer, stdout: term.OutputRaw()},
+		printer: printer,
 	}
 
 	if opts.BlobID {
@@ -617,8 +671,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		}
 	}
 
-	var filteredSnapshots []*restic.Snapshot
+	var filteredSnapshots []*data.Snapshot
-	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots, printer) {
 		filteredSnapshots = append(filteredSnapshots, sn)
 	}
 	if ctx.Err() != nil {
@@ -626,7 +680,10 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	}
 
 	sort.Slice(filteredSnapshots, func(i, j int) bool {
-		return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
+		if opts.Reverse {
+			return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
+		}
+		return filteredSnapshots[i].Time.After(filteredSnapshots[j].Time)
 	})
 
 	for _, sn := range filteredSnapshots {

cmd/restic/cmd_find_integration_test.go

@@ -3,19 +3,23 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/global"
+	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui"
 )
 
-func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern string) []byte {
+func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts global.Options, pattern string) []byte {
-	buf, err := withCaptureStdout(func() error {
+	buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
 		gopts.JSON = wantJSON
 
-		opts := FindOptions{}
+		return runFind(ctx, opts, gopts, []string{pattern}, gopts.Term)
-		return runFind(context.TODO(), opts, gopts, []string{pattern})
 	})
 	rtest.OK(t, err)
 	return buf.Bytes()
@@ -29,16 +33,15 @@ func TestFind(t *testing.T) {
 	opts := BackupOptions{}
 
 	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
-	testRunCheck(t, env.gopts)
 
-	results := testRunFind(t, false, env.gopts, "unexistingfile")
+	results := testRunFind(t, false, FindOptions{}, env.gopts, "unexistingfile")
 	rtest.Assert(t, len(results) == 0, "unexisting file found in repo (%v)", datafile)
 
-	results = testRunFind(t, false, env.gopts, "testfile")
+	results = testRunFind(t, false, FindOptions{}, env.gopts, "testfile")
 	lines := strings.Split(string(results), "\n")
 	rtest.Assert(t, len(lines) == 2, "expected one file found in repo (%v)", datafile)
 
-	results = testRunFind(t, false, env.gopts, "testfile*")
+	results = testRunFind(t, false, FindOptions{}, env.gopts, "testfile*")
 	lines = strings.Split(string(results), "\n")
 	rtest.Assert(t, len(lines) == 4, "expected three files found in repo (%v)", datafile)
 }
@@ -67,21 +70,207 @@ func TestFindJSON(t *testing.T) {
 
 	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
+	snapshot, _ := testRunSnapshots(t, env.gopts)
 
-	results := testRunFind(t, true, env.gopts, "unexistingfile")
+	results := testRunFind(t, true, FindOptions{}, env.gopts, "unexistingfile")
 	matches := []testMatches{}
 	rtest.OK(t, json.Unmarshal(results, &matches))
 	rtest.Assert(t, len(matches) == 0, "expected no match in repo (%v)", datafile)
 
-	results = testRunFind(t, true, env.gopts, "testfile")
+	results = testRunFind(t, true, FindOptions{}, env.gopts, "testfile")
 	rtest.OK(t, json.Unmarshal(results, &matches))
 	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
 	rtest.Assert(t, len(matches[0].Matches) == 1, "expected a single file to match (%v)", datafile)
 	rtest.Assert(t, matches[0].Hits == 1, "expected hits to show 1 match (%v)", datafile)
 
-	results = testRunFind(t, true, env.gopts, "testfile*")
+	results = testRunFind(t, true, FindOptions{}, env.gopts, "testfile*")
 	rtest.OK(t, json.Unmarshal(results, &matches))
 	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", datafile)
 	rtest.Assert(t, len(matches[0].Matches) == 3, "expected 3 files to match (%v)", datafile)
 	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
+
+	results = testRunFind(t, true, FindOptions{TreeID: true}, env.gopts, snapshot.Tree.String())
+	rtest.OK(t, json.Unmarshal(results, &matches))
+	rtest.Assert(t, len(matches) == 1, "expected a single snapshot in repo (%v)", matches)
+	rtest.Assert(t, len(matches[0].Matches) == 3, "expected 3 files to match (%v)", matches[0].Matches)
+	rtest.Assert(t, matches[0].Hits == 3, "expected hits to show 3 matches (%v)", datafile)
+}
+
+func TestFindSorting(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+
+	// first backup
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	sn1 := testListSnapshots(t, env.gopts, 1)[0]
+
+	// second backup
+	testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
+	snapshots := testListSnapshots(t, env.gopts, 2)
+	// get id of new snapshot without depending on file order returned by filesystem
+	sn2 := snapshots[0]
+	if sn1.Equal(sn2) {
+		sn2 = snapshots[1]
+	}
+
+	// first restic find - with default FindOptions{}
+	results := testRunFind(t, true, FindOptions{}, env.gopts, "testfile")
+	lines := strings.Split(string(results), "\n")
+	rtest.Assert(t, len(lines) == 2, "expected two lines of output, found %d", len(lines))
+	matches := []testMatches{}
+	rtest.OK(t, json.Unmarshal(results, &matches))
+
+	// run second restic find with --reverse, sort oldest to newest
+	resultsReverse := testRunFind(t, true, FindOptions{Reverse: true}, env.gopts, "testfile")
+	lines = strings.Split(string(resultsReverse), "\n")
+	rtest.Assert(t, len(lines) == 2, "expected two lines of output, found %d", len(lines))
+	matchesReverse := []testMatches{}
+	rtest.OK(t, json.Unmarshal(resultsReverse, &matchesReverse))
+
+	// compare result sets
+	rtest.Assert(t, sn1.String() == matchesReverse[0].SnapshotID, "snapshot[0] must match old snapshot")
+	rtest.Assert(t, sn2.String() == matchesReverse[1].SnapshotID, "snapshot[1] must match new snapshot")
+	rtest.Assert(t, matches[0].SnapshotID == matchesReverse[1].SnapshotID, "matches should be sorted 1")
+	rtest.Assert(t, matches[1].SnapshotID == matchesReverse[0].SnapshotID, "matches should be sorted 2")
+}
+
+func TestFindInvalidTimeRange(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	err := runFind(context.TODO(), FindOptions{Oldest: "2026-01-01", Newest: "2020-01-01"}, env.gopts, []string{"quack"}, env.gopts.Term)
+	rtest.Assert(t, err != nil && err.Error() == "Fatal: --oldest must specify a time before --newest",
+		"unexpected error message: %v", err)
+}
+
+// JsonOutput is the struct `restic find --json` produces
+type JSONOutput struct {
+	ObjectType string    `json:"object_type"`
+	ID         string    `json:"id"`
+	Path       string    `json:"path"`
+	ParentTree string    `json:"parent_tree,omitempty"`
+	SnapshotID string    `json:"snapshot"`
+	Time       time.Time `json:"time,omitempty"`
+}
+
+func TestFindPackfile(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	// backup
+	backupPath := env.testdata + "/0/0/9"
+	testRunBackup(t, "", []string{backupPath}, BackupOptions{}, env.gopts)
+	sn1 := testListSnapshots(t, env.gopts, 1)[0]
+
+	// do all the testing wrapped inside withTermStatus()
+	err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
+		printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
+		_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
+		rtest.OK(t, err)
+		defer unlock()
+
+		// load master index
+		rtest.OK(t, repo.LoadIndex(ctx, printer))
+
+		packID := restic.ID{}
+		done := false
+		err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) {
+			if !done && pb.Type == restic.TreeBlob {
+				packID = pb.PackID
+				done = true
+			}
+		})
+
+		rtest.OK(t, err)
+		rtest.Assert(t, !packID.IsNull(), "expected a tree packfile ID")
+		findOptions := FindOptions{PackID: true}
+		results := testRunFind(t, true, findOptions, env.gopts, packID.String())
+
+		// get the json records
+		jsonResult := []JSONOutput{}
+		rtest.OK(t, json.Unmarshal(results, &jsonResult))
+		rtest.Assert(t, len(jsonResult) > 0, "expected at least one tree record in the packfile")
+
+		// look at the last record
+		lastIndex := len(jsonResult) - 1
+		record := jsonResult[lastIndex]
+		rtest.Assert(t, record.ObjectType == "tree" && record.SnapshotID == sn1.String(),
+			"expected a tree record with known snapshot id, but got type=%s and snapID=%s instead of %s",
+			record.ObjectType, record.SnapshotID, sn1.String())
+		backupPath = filepath.ToSlash(backupPath)[2:] // take the offending drive mapping away
+		rtest.Assert(t, strings.Contains(record.Path, backupPath), "expected %q as part of %q", backupPath, record.Path)
+
+		return nil
+	})
+	rtest.OK(t, err)
+}
+
+func TestFindPackID(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	testSetupBackupData(t, env)
+
+	dir009 := filepath.Join(env.testdata, "0", "0", "9")
+	dirEntries, err := os.ReadDir(dir009)
+	rtest.OK(t, err)
+	numberOfFiles := len(dirEntries)
+
+	// backup
+	testRunBackup(t, "", []string{dir009}, BackupOptions{}, env.gopts)
+	sn1 := testListSnapshots(t, env.gopts, 1)[0]
+
+	// extract packfile ID from repository index
+	dataPackID := restic.ID{}
+	treePackID := restic.ID{}
+	err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
+		printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
+		_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
+		rtest.OK(t, err)
+		defer unlock()
+
+		// load Index
+		rtest.OK(t, repo.LoadIndex(ctx, nil))
+		// go through all index entries and collect data and tree packfile(s)
+		rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
+			switch blob.Type {
+			case restic.DataBlob:
+				dataPackID = blob.PackID
+			case restic.TreeBlob:
+				treePackID = blob.PackID
+			}
+		}))
+		return nil
+	})
+	rtest.OK(t, err)
+
+	// look for data packfile
+	rtest.Assert(t, !dataPackID.IsNull(), "expected to find data packfile in repo")
+	packID := dataPackID.String()
+	out := testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID)
+
+	findRes := []JSONOutput{}
+	rtest.OK(t, json.Unmarshal(out, &findRes))
+	rtest.Assert(t, len(findRes) == numberOfFiles, "expected %d entries for this packfile, got %d",
+		numberOfFiles, len(findRes))
+
+	// look for tree packfile
+	rtest.Assert(t, !treePackID.IsNull(), "expected to find tree packfile in repo")
+	packID = treePackID.String()
+	out = testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID)
+
+	findRes = []JSONOutput{}
+	rtest.OK(t, json.Unmarshal(out, &findRes))
+	record := findRes[len(findRes)-1]
+
+	rtest.Equals(t, record.ObjectType, "tree")
+	rtest.Equals(t, record.SnapshotID, sn1.String())
+	// windows path are messy, so we get rid of the messy bits at the start
+	// exp: "/C/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9"
+	// got: "C:/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9"
+	rtest.Equals(t, filepath.ToSlash(record.Path)[2:], filepath.ToSlash(dir009)[2:])
 }