Add version for 0.12.0

changelog: Correct and slightly polish some unreleased changelog files

.github/workflows/tests.yml

@@ -0,0 +1,258 @@
+name: test
+on:
+  # run tests on push to master, but not when other branches are pushed to
+  push:
+    branches:
+      - master
+
+  # run tests for all pull requests
+  pull_request:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        # list of jobs to run:
+        include:
+          - job_name: Windows
+            go: 1.15.x
+            os: windows-latest
+
+          - job_name: macOS
+            go: 1.15.x
+            os: macOS-latest
+            test_fuse: false
+
+          - job_name: Linux
+            go: 1.15.x
+            os: ubuntu-latest
+            test_cloud_backends: true
+            test_fuse: true
+            check_changelog: true
+
+          - job_name: Linux
+            go: 1.14.x
+            os: ubuntu-latest
+            test_fuse: true
+
+          - job_name: Linux
+            go: 1.13.x
+            os: ubuntu-latest
+            test_fuse: true
+
+    name: ${{ matrix.job_name }} Go ${{ matrix.go }}
+    runs-on: ${{ matrix.os }}
+
+    env:
+      GOPROXY: https://proxy.golang.org
+
+    steps:
+      - name: Set up Go ${{ matrix.go }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Get programs (Linux/macOS)
+        run: |
+          echo "build Go tools"
+          go get github.com/restic/rest-server/...
+
+          echo "install minio server"
+          mkdir $HOME/bin
+          if [ "$RUNNER_OS" == "macOS" ]; then
+            wget --no-verbose -O $HOME/bin/minio https://dl.minio.io/server/minio/release/darwin-amd64/minio
+          else
+            wget --no-verbose -O $HOME/bin/minio https://dl.minio.io/server/minio/release/linux-amd64/minio
+          fi
+          chmod 755 $HOME/bin/minio
+
+          echo "install rclone"
+          if [ "$RUNNER_OS" == "macOS" ]; then
+            wget --no-verbose -O rclone.zip https://downloads.rclone.org/rclone-current-osx-amd64.zip
+          else
+            wget --no-verbose -O rclone.zip https://downloads.rclone.org/rclone-current-linux-amd64.zip
+          fi
+          unzip rclone.zip
+          cp rclone*/rclone $HOME/bin
+          chmod 755 $HOME/bin/rclone
+          rm -rf rclone*
+
+          # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          echo $HOME/bin >> $GITHUB_PATH
+        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
+
+      - name: Get programs (Windows)
+        shell: powershell
+        run: |
+          $ProgressPreference = 'SilentlyContinue'
+
+          echo "build Go tools"
+          go get github.com/restic/rest-server/...
+
+          echo "install minio server"
+          mkdir $Env:USERPROFILE/bin
+          Invoke-WebRequest https://dl.minio.io/server/minio/release/windows-amd64/minio.exe -OutFile $Env:USERPROFILE/bin/minio.exe
+
+          echo "install rclone"
+          Invoke-WebRequest https://downloads.rclone.org/rclone-current-windows-amd64.zip -OutFile rclone.zip
+
+          unzip rclone.zip
+          copy rclone*/rclone.exe $Env:USERPROFILE/bin
+
+          # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v2)
+          echo $Env:USERPROFILE\bin >> $Env:GITHUB_PATH
+
+          echo "install tar"
+          cd $env:USERPROFILE
+          mkdir tar
+          cd tar
+
+          # install exactly these versions of tar and the libraries, other combinations might not work!
+
+          Invoke-WebRequest https://github.com/restic/test-assets/raw/master/tar-1.13-1-bin.zip -OutFile tar.zip
+          unzip tar.zip
+          Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libintl-0.11.5-2-bin.zip -OutFile libintl.zip
+          unzip libintl.zip
+          Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libiconv-1.8-1-bin.zip -OutFile libiconv.zip
+          unzip libiconv.zip
+
+          # add $USERPROFILE/tar/bin to path
+          echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH
+        if: matrix.os == 'windows-latest'
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Build with build.go
+        run: |
+          go run build.go
+
+      - name: Run local Tests
+        env:
+          RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
+        run: |
+          go test -cover ./...
+
+      - name: Test cloud backends
+        env:
+          RESTIC_TEST_S3_KEY: ${{ secrets.RESTIC_TEST_S3_KEY }}
+          RESTIC_TEST_S3_SECRET: ${{ secrets.RESTIC_TEST_S3_SECRET }}
+          RESTIC_TEST_S3_REPOSITORY: ${{ secrets.RESTIC_TEST_S3_REPOSITORY }}
+          RESTIC_TEST_AZURE_ACCOUNT_NAME: ${{ secrets.RESTIC_TEST_AZURE_ACCOUNT_NAME }}
+          RESTIC_TEST_AZURE_ACCOUNT_KEY: ${{ secrets.RESTIC_TEST_AZURE_ACCOUNT_KEY }}
+          RESTIC_TEST_AZURE_REPOSITORY: ${{ secrets.RESTIC_TEST_AZURE_REPOSITORY }}
+          RESTIC_TEST_B2_ACCOUNT_ID: ${{ secrets.RESTIC_TEST_B2_ACCOUNT_ID }}
+          RESTIC_TEST_B2_ACCOUNT_KEY: ${{ secrets.RESTIC_TEST_B2_ACCOUNT_KEY }}
+          RESTIC_TEST_B2_REPOSITORY: ${{ secrets.RESTIC_TEST_B2_REPOSITORY }}
+          RESTIC_TEST_GS_REPOSITORY: ${{ secrets.RESTIC_TEST_GS_REPOSITORY }}
+          RESTIC_TEST_GS_PROJECT_ID: ${{ secrets.RESTIC_TEST_GS_PROJECT_ID }}
+          GOOGLE_PROJECT_ID: ${{ secrets.RESTIC_TEST_GS_PROJECT_ID }}
+          RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64: ${{ secrets.RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 }}
+          RESTIC_TEST_OS_AUTH_URL: ${{ secrets.RESTIC_TEST_OS_AUTH_URL }}
+          RESTIC_TEST_OS_TENANT_NAME: ${{ secrets.RESTIC_TEST_OS_TENANT_NAME }}
+          RESTIC_TEST_OS_USERNAME: ${{ secrets.RESTIC_TEST_OS_USERNAME }}
+          RESTIC_TEST_OS_PASSWORD: ${{ secrets.RESTIC_TEST_OS_PASSWORD }}
+          RESTIC_TEST_OS_REGION_NAME: ${{ secrets.RESTIC_TEST_OS_REGION_NAME }}
+          RESTIC_TEST_SWIFT: ${{ secrets.RESTIC_TEST_SWIFT }}
+          # fail if any of the following tests cannot be run
+          RESTIC_TEST_DISALLOW_SKIP: "restic/backend/rest.TestBackendREST,\
+            restic/backend/sftp.TestBackendSFTP,\
+            restic/backend/s3.TestBackendMinio,\
+            restic/backend/rclone.TestBackendRclone,\
+            restic/backend/s3.TestBackendS3,\
+            restic/backend/swift.TestBackendSwift,\
+            restic/backend/b2.TestBackendB2,\
+            restic/backend/gs.TestBackendGS,\
+            restic/backend/azure.TestBackendAzure"
+        run: |
+          # prepare credentials for Google Cloud Storage tests in a temp file
+          export GOOGLE_APPLICATION_CREDENTIALS=$(mktemp --tmpdir restic-gcs-auth-XXXXXXX)
+          echo $RESTIC_TEST_GS_APPLICATION_CREDENTIALS_B64 | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
+          go test -cover -parallel 4 ./internal/backend/...
+
+        # only run cloud backend tests for pull requests from and pushes to our
+        # own repo, otherwise the secrets are not available
+        if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && matrix.test_cloud_backends
+
+      - name: Check changelog files with calens
+        run: |
+          echo "install calens"
+          go get github.com/restic/calens
+
+          echo "check changelog files"
+          calens
+        if: matrix.check_changelog
+
+  cross_compile:
+    strategy:
+
+      # ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
+      matrix:
+        # run cross-compile in two batches parallel so the overall tests run faster
+        targets:
+          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le \
+            openbsd/386 openbsd/amd64"
+
+          - "freebsd/386 freebsd/amd64 freebsd/arm \
+            aix/ppc64 \
+            darwin/amd64 \
+            netbsd/386 netbsd/amd64 \
+            windows/386 windows/amd64 \
+            solaris/amd64"
+
+    env:
+      go: 1.15.x
+      GOPROXY: https://proxy.golang.org
+
+    runs-on: ubuntu-latest
+
+    name: Cross Compile for ${{ matrix.targets }}
+
+    steps:
+      - name: Set up Go ${{ env.go }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.go }}
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install gox
+        run: |
+          go get github.com/mitchellh/gox
+
+      - name: Cross-compile with gox for ${{ matrix.targets }}
+        env:
+          GOFLAGS: "-trimpath"
+          GOX_ARCHS: "${{ matrix.targets }}"
+        run: |
+          mkdir build-output
+          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}" ./cmd/restic
+          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -tags debug -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}_debug" ./cmd/restic
+
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.36
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          only-new-issues: true
+          args: --verbose --timeout 5m
+
+        # only run golangci-lint for pull requests, otherwise ALL hints get
+        # reported. We need to slowly address all issues until we can enable
+        # linting the master branch :)
+        if: github.event_name == 'pull_request'
+
+      - name: Check go.mod/go.sum
+        run: |
+          echo "check if go.mod and go.sum are up to date"
+          go mod tidy
+          git diff --exit-code go.mod go.sum

.gitignore

@@ -1,2 +1,3 @@
 /restic
 /.vagrant
+/.vscode

.golangci.yml

@@ -0,0 +1,57 @@
+# This is the configuration for golangci-lint for the restic project.
+#
+# A sample config with all settings is here:
+# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
+
+linters:
+  # only enable the linters listed below
+  disable-all: true
+  enable:
+    # make sure all errors returned by functions are handled
+    - errcheck
+
+    # find unused code
+    - deadcode
+
+    # show how code can be simplified
+    - gosimple
+
+    # # make sure code is formatted
+    - gofmt
+
+    # examine code and report suspicious constructs, such as Printf calls whose
+    # arguments do not align with the format string
+    - govet
+
+    # make sure names and comments are used according to the conventions
+    - golint
+
+    # detect when assignments to existing variables are not used
+    - ineffassign
+
+    # run static analysis and find errors
+    - staticcheck
+
+    # find unused variables, functions, structs, types, etc.
+    - unused
+
+    # find unused struct fields
+    - structcheck
+
+    # find unused global variables
+    - varcheck
+
+    # parse and typecheck code
+    - typecheck
+
+issues:
+  # don't use the default exclude rules, this hides (among others) ignored
+  # errors from Close() calls
+  exclude-use-default: false
+
+  # list of things to not warn about
+  exclude:
+    # golint: do not warn about missing comments for exported stuff
+    - exported (function|method|var|type|const) `.*` should have comment or be unexported
+    # golint: ignore constants in all caps
+    - don't use ALL_CAPS in Go names; use CamelCase

.hound.yml

@@ -1,2 +0,0 @@
-go:
-  enabled: true

.travis.yml

@@ -1,58 +0,0 @@
-language: go
-sudo: false
-
-matrix:
-  include:
-    - os: linux
-      go: "1.13.x"
-      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
-      cache:
-        directories:
-          - $HOME/.cache/go-build
-          - $HOME/gopath/pkg/mod
-
-    - os: linux
-      go: "1.14.x"
-      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
-      cache:
-        directories:
-          - $HOME/.cache/go-build
-          - $HOME/gopath/pkg/mod
-
-    # only run fuse and cloud backends tests on Travis for the latest Go on Linux
-    - os: linux
-      go: "1.15.x"
-      sudo: true
-      cache:
-        directories:
-          - $HOME/.cache/go-build
-          - $HOME/gopath/pkg/mod
-
-    - os: osx
-      go: "1.15.x"
-      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
-      cache:
-        directories:
-          - $HOME/Library/Caches/go-build
-          - $HOME/gopath/pkg/mod
-
-branches:
-  only:
-    - master
-
-notifications:
-  irc:
-    channels:
-      - "chat.freenode.net#restic"
-    on_success: change
-    on_failure: change
-    skip_join: true
-
-install:
-  - go version
-  - export GOBIN="$GOPATH/bin"
-  - export PATH="$PATH:$GOBIN"
-  - go env
-
-script:
-  - go run run_integration_tests.go

CHANGELOG.md

@@ -1,3 +1,675 @@
+Changelog for restic 0.12.0 (2021-02-14)
+=======================================
+
+The following sections list the changes in restic 0.12.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1681: Make `mount` not create missing mount point directory
+ * Fix #1800: Ignore `no data available` filesystem error during backup
+ * Fix #2563: Report the correct owner of directories in FUSE mounts
+ * Fix #2688: Make `backup` and `tag` commands separate tags by comma
+ * Fix #2739: Make the `cat` command respect the `--no-lock` option
+ * Fix #3087: The `--use-fs-snapshot` option now works on windows/386
+ * Fix #3100: Do not require gs bucket permissions when running `init`
+ * Fix #3111: Correctly detect output redirection for `backup` command on Windows
+ * Fix #3151: Don't create invalid snapshots when `backup` is interrupted
+ * Fix #3166: Improve error handling in the `restore` command
+ * Fix #3232: Correct statistics for overlapping targets
+ * Fix #3014: Fix sporadic stream reset between rclone and restic
+ * Fix #3152: Do not hang until foregrounded when completed in background
+ * Fix #3249: Improve error handling in `gs` backend
+ * Chg #3095: Deleting files on Google Drive now moves them to the trash
+ * Enh #2186: Allow specifying percentage in `check --read-data-subset`
+ * Enh #2453: Report permanent/fatal backend errors earlier
+ * Enh #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
+ * Enh #2706: Configurable progress reports for non-interactive terminals
+ * Enh #2944: Add `backup` options `--files-from-{verbatim,raw}`
+ * Enh #3083: Allow usage of deprecated S3 `ListObjects` API
+ * Enh #3147: Support additional environment variables for Swift authentication
+ * Enh #3191: Add release binaries for MIPS architectures
+ * Enh #909: Back up mountpoints as empty directories
+ * Enh #3250: Add several more error checks
+ * Enh #2718: Improve `prune` performance and make it more customizable
+ * Enh #2495: Add option to let `backup` trust mtime without checking ctime
+ * Enh #2941: Speed up the repacking step of the `prune` command
+ * Enh #3006: Speed up the `rebuild-index` command
+ * Enh #3048: Add more checks for index and pack files in the `check` command
+ * Enh #2433: Make the `dump` command support `zip` format
+ * Enh #3099: Reduce memory usage of `check` command
+ * Enh #3106: Parallelize scan of snapshot content in `copy` and `prune`
+ * Enh #3130: Parallelize reading of locks and snapshots
+ * Enh #3254: Enable HTTP/2 for backend connections
+
+Details
+-------
+
+ * Bugfix #1681: Make `mount` not create missing mount point directory
+
+   When specifying a non-existent directory as mount point for the `mount` command, restic used
+   to create the specified directory automatically.
+
+   This has now changed such that restic instead gives an error when the specified directory for
+   the mount point does not exist.
+
+   https://github.com/restic/restic/issues/1681
+   https://github.com/restic/restic/pull/3008
+
+ * Bugfix #1800: Ignore `no data available` filesystem error during backup
+
+   Restic was unable to backup files on some filesystems, for example certain configurations of
+   CIFS on Linux which return a `no data available` error when reading extended attributes. These
+   errors are now ignored.
+
+   https://github.com/restic/restic/issues/1800
+   https://github.com/restic/restic/pull/3034
+
+ * Bugfix #2563: Report the correct owner of directories in FUSE mounts
+
+   Restic 0.10.0 changed the FUSE mount to always report the current user as the owner of
+   directories within the FUSE mount, which is incorrect.
+
+   This is now changed back to reporting the correct owner of a directory.
+
+   https://github.com/restic/restic/issues/2563
+   https://github.com/restic/restic/pull/3141
+
+ * Bugfix #2688: Make `backup` and `tag` commands separate tags by comma
+
+   Running `restic backup --tag foo,bar` previously created snapshots with one single tag
+   containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`).
+
+   Similarly, the `tag` command's `--set`, `--add` and `--remove` options would treat
+   `foo,bar` as one tag instead of two tags. This was inconsistent with other commands and often
+   unexpected when one intended `foo,bar` to mean two tags.
+
+   To be consistent in all commands, restic now interprets `foo,bar` to mean two separate tags
+   (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, including in the `backup` and
+   `tag` commands.
+
+   NOTE: This change might result in unexpected behavior in cases where you use the `forget`
+   command and filter on tags like `foo,bar`. Snapshots previously backed up with `--tag
+   foo,bar` will still not match that filter, but snapshots saved from now on will match that
+   filter.
+
+   To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can first generate a
+   list of the relevant snapshots using a command like:
+
+   Restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id'
+
+   And then use `restic tag --set foo --set bar snapshotID [...]` to set the new tags. Please adjust
+   the commands to include real tag names and any additional tags, as well as the list of snapshots
+   to process.
+
+   https://github.com/restic/restic/issues/2688
+   https://github.com/restic/restic/pull/2690
+   https://github.com/restic/restic/pull/3197
+
+ * Bugfix #2739: Make the `cat` command respect the `--no-lock` option
+
+   The `cat` command would not respect the `--no-lock` flag. This is now fixed.
+
+   https://github.com/restic/restic/issues/2739
+
+ * Bugfix #3087: The `--use-fs-snapshot` option now works on windows/386
+
+   Restic failed to create VSS snapshots on windows/386 with the following error:
+
+   GetSnapshotProperties() failed: E_INVALIDARG (0x80070057)
+
+   This is now fixed.
+
+   https://github.com/restic/restic/issues/3087
+   https://github.com/restic/restic/pull/3090
+
+ * Bugfix #3100: Do not require gs bucket permissions when running `init`
+
+   Restic used to require bucket level permissions for the `gs` backend in order to initialize a
+   restic repository.
+
+   It now allows a `gs` service account to initialize a repository if the bucket does exist and the
+   service account has permissions to write/read to that bucket.
+
+   https://github.com/restic/restic/issues/3100
+
+ * Bugfix #3111: Correctly detect output redirection for `backup` command on Windows
+
+   On Windows, since restic 0.10.0 the `backup` command did not properly detect when the output
+   was redirected to a file. This caused restic to output terminal control characters. This has
+   been fixed by correcting the terminal detection.
+
+   https://github.com/restic/restic/issues/3111
+   https://github.com/restic/restic/pull/3150
+
+ * Bugfix #3151: Don't create invalid snapshots when `backup` is interrupted
+
+   When canceling a backup run at a certain moment it was possible that restic created a snapshot
+   with an invalid "null" tree. This caused `check` and other operations to fail. The `backup`
+   command now properly handles interruptions and never saves a snapshot when interrupted.
+
+   https://github.com/restic/restic/issues/3151
+   https://github.com/restic/restic/pull/3164
+
+ * Bugfix #3166: Improve error handling in the `restore` command
+
+   The `restore` command used to not print errors while downloading file contents from the
+   repository. It also incorrectly exited with a zero error code even when there were errors
+   during the restore process. This has all been fixed and `restore` now returns with a non-zero
+   exit code when there's an error.
+
+   https://github.com/restic/restic/issues/3166
+   https://github.com/restic/restic/pull/3207
+
+ * Bugfix #3232: Correct statistics for overlapping targets
+
+   A user reported that restic's statistics and progress information during backup was not
+   correctly calculated when the backup targets (files/dirs to save) overlap. For example,
+   consider a directory `foo` which contains (among others) a file `foo/bar`. When `restic
+   backup foo foo/bar` was run, restic counted the size of the file `foo/bar` twice, so the
+   completeness percentage as well as the number of files was wrong. This is now corrected.
+
+   https://github.com/restic/restic/issues/3232
+   https://github.com/restic/restic/pull/3243
+
+ * Bugfix #3014: Fix sporadic stream reset between rclone and restic
+
+   Sometimes when using restic with the `rclone` backend, an error message similar to the
+   following would be printed:
+
+   Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
+
+   It was found that this was caused by restic closing the connection to rclone to soon when
+   downloading data. A workaround has been added which waits for the end of the download before
+   closing the connection.
+
+   https://github.com/rclone/rclone/issues/2598
+   https://github.com/restic/restic/pull/3014
+
+ * Bugfix #3152: Do not hang until foregrounded when completed in background
+
+   On Linux, when running in the background restic failed to stop the terminal output of the
+   `backup` command after it had completed. This caused restic to hang until moved to the
+   foreground. This has now been fixed.
+
+   https://github.com/restic/restic/pull/3152
+   https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334
+
+ * Bugfix #3249: Improve error handling in `gs` backend
+
+   The `gs` backend did not notice when the last step of completing a file upload failed. Under rare
+   circumstances, this could cause missing files in the backup repository. This has now been
+   fixed.
+
+   https://github.com/restic/restic/pull/3249
+
+ * Change #3095: Deleting files on Google Drive now moves them to the trash
+
+   When deleting files on Google Drive via the `rclone` backend, restic used to bypass the trash
+   folder required that one used the `-o rclone.args` option to enable usage of the trash folder.
+   This ensured that deleted files in Google Drive were not kept indefinitely in the trash folder.
+   However, since Google Drive's trash retention policy changed to deleting trashed files after
+   30 days, this is no longer needed.
+
+   Restic now leaves it up to rclone and its configuration to use or not use the trash folder when
+   deleting files. The default is to use the trash folder, as of rclone 1.53.2. To re-enable the
+   restic 0.11 behavior, set the `RCLONE_DRIVE_USE_TRASH` environment variable or change the
+   rclone configuration. See the rclone documentation for more details.
+
+   https://github.com/restic/restic/issues/3095
+   https://github.com/restic/restic/pull/3102
+
+ * Enhancement #2186: Allow specifying percentage in `check --read-data-subset`
+
+   We've enhanced the `check` command's `--read-data-subset` option to also accept a
+   percentage (e.g. `2.5%` or `10%`). This will check the given percentage of pack files (which
+   are randomly selected on each run).
+
+   https://github.com/restic/restic/issues/2186
+   https://github.com/restic/restic/pull/3038
+
+ * Enhancement #2453: Report permanent/fatal backend errors earlier
+
+   When encountering errors in reading from or writing to storage backends, restic retries the
+   failing operation up to nine times (for a total of ten attempts). It used to retry all backend
+   operations, but now detects some permanent error conditions so that it can report fatal errors
+   earlier.
+
+   Permanent failures include local disks being full, SSH connections dropping and permission
+   errors.
+
+   https://github.com/restic/restic/issues/2453
+   https://github.com/restic/restic/issues/3180
+   https://github.com/restic/restic/pull/3170
+   https://github.com/restic/restic/pull/3181
+
+ * Enhancement #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
+
+   A new extended option `s3.bucket-lookup` has been added to support Alibaba/Aliyun OSS in the
+   `s3` backend. The option can be set to one of the following values:
+
+   - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - Use path style
+   bucket access
+
+   To make the `s3` backend work with Alibaba/Aliyun OSS you must set `s3.bucket-lookup` to `dns`
+   and set the `s3.region` parameter. For example:
+
+   Restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r
+   s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
+
+   Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up and it seems that
+   Alibaba doesn't support that properly.
+
+   https://github.com/restic/restic/issues/2528
+   https://github.com/restic/restic/pull/2535
+
+ * Enhancement #2706: Configurable progress reports for non-interactive terminals
+
+   The `backup`, `check` and `prune` commands never printed any progress reports on
+   non-interactive terminals. This behavior is now configurable using the
+   `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for an update
+   every second, or `0.01666` for an update every minute.
+
+   The `backup` command now also prints the current progress when restic receives a `SIGUSR1`
+   signal.
+
+   Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` signal
+   prints a status report even when `--quiet` was specified.
+
+   https://github.com/restic/restic/issues/2706
+   https://github.com/restic/restic/issues/3194
+   https://github.com/restic/restic/pull/3199
+
+ * Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}`
+
+   The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of
+   files to back up from a file. Unlike the existing `--files-from` option, these options do not
+   interpret the listed filenames as glob patterns; instead, whitespace in filenames is
+   preserved as-is and no pattern expansion is done. Please see the documentation for specifics.
+
+   These new options are highly recommended over `--files-from`, when using a script to generate
+   the list of files to back up.
+
+   https://github.com/restic/restic/issues/2944
+   https://github.com/restic/restic/issues/3013
+
+ * Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API
+
+   Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2`
+   implementation which causes problems for restic when using their API endpoints. When a broken
+   server implementation is used, restic prints errors similar to the following:
+
+   List() returned error: Truncated response should have continuation token set
+
+   As a temporary workaround, restic now allows using the older `ListObjects` endpoint by
+   setting the `s3.list-objects-v1` extended option, for instance:
+
+   Restic -o s3.list-objects-v1=true snapshots
+
+   Please note that this option may be removed in future versions of restic.
+
+   https://github.com/restic/restic/issues/3083
+   https://github.com/restic/restic/pull/3085
+
+ * Enhancement #3147: Support additional environment variables for Swift authentication
+
+   The `swift` backend now supports the following additional environment variables for passing
+   authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`,
+   `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
+
+   Depending on the `openrc` configuration file these might be required when the user and project
+   domains differ from one another.
+
+   https://github.com/restic/restic/issues/3147
+   https://github.com/restic/restic/pull/3158
+
+ * Enhancement #3191: Add release binaries for MIPS architectures
+
+   We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`,
+   `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems.
+
+   https://github.com/restic/restic/issues/3191
+   https://github.com/restic/restic/pull/3208
+
+ * Enhancement #909: Back up mountpoints as empty directories
+
+   When the `--one-file-system` option is specified to `restic backup`, it ignores all file
+   systems mounted below one of the target directories. This means that when a snapshot is
+   restored, users needed to manually recreate the mountpoint directories.
+
+   Restic now backs up mountpoints as empty directories and therefore implements the same
+   approach as `tar`.
+
+   https://github.com/restic/restic/issues/909
+   https://github.com/restic/restic/pull/3119
+
+ * Enhancement #3250: Add several more error checks
+
+   We've added a lot more error checks in places where errors were previously ignored (as hinted by
+   the static analysis program `errcheck` via `golangci-lint`).
+
+   https://github.com/restic/restic/pull/3250
+
+ * Enhancement #2718: Improve `prune` performance and make it more customizable
+
+   The `prune` command is now much faster. This is especially the case for remote repositories or
+   repositories with not much data to remove. Also the memory usage of the `prune` command is now
+   reduced.
+
+   Restic used to rebuild the index from scratch after pruning. This could lead to missing packs in
+   the index in some cases for eventually consistent backends such as e.g. AWS S3. This behavior is
+   now changed and the index rebuilding uses the information already known by `prune`.
+
+   By default, the `prune` command no longer removes all unused data. This behavior can be
+   fine-tuned by new options, like the acceptable amount of unused space or the maximum size of
+   data to reorganize. For more details, please see
+   https://restic.readthedocs.io/en/stable/060_forget.html .
+
+   Moreover, `prune` now accepts the `--dry-run` option and also running `forget --dry-run
+   --prune` will show what `prune` would do.
+
+   This enhancement also fixes several open issues, e.g.: -
+   https://github.com/restic/restic/issues/1140 -
+   https://github.com/restic/restic/issues/1599 -
+   https://github.com/restic/restic/issues/1985 -
+   https://github.com/restic/restic/issues/2112 -
+   https://github.com/restic/restic/issues/2227 -
+   https://github.com/restic/restic/issues/2305
+
+   https://github.com/restic/restic/pull/2718
+   https://github.com/restic/restic/pull/2842
+
+ * Enhancement #2495: Add option to let `backup` trust mtime without checking ctime
+
+   The `backup` command used to require that both `ctime` and `mtime` of a file matched with a
+   previously backed up version to determine that the file was unchanged. In other words, if
+   either `ctime` or `mtime` of the file had changed, it would be considered changed and restic
+   would read the file's content again to back up the relevant (changed) parts of it.
+
+   The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes
+   for a file does not cause restic to read the file's contents again.
+
+   The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more
+   reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned
+   out to often be expensive because it made restic read file contents even if only the metadata
+   (owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user
+   restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned
+   off this behavior, but also removed a different check.
+
+   Please note that changes in files' metadata are still recorded, regardless of the command line
+   options provided to the backup command.
+
+   https://github.com/restic/restic/issues/2495
+   https://github.com/restic/restic/issues/2558
+   https://github.com/restic/restic/issues/2819
+   https://github.com/restic/restic/pull/2823
+
+ * Enhancement #2941: Speed up the repacking step of the `prune` command
+
+   The repack step of the `prune` command, which moves still used file parts into new pack files
+   such that the old ones can be garbage collected later on, now processes multiple pack files in
+   parallel. This is especially beneficial for high latency backends or when using a fast network
+   connection.
+
+   https://github.com/restic/restic/pull/2941
+
+ * Enhancement #3006: Speed up the `rebuild-index` command
+
+   We've optimized the `rebuild-index` command. Now, existing index entries are used to
+   minimize the number of pack files that must be read. This speeds up the index rebuild a lot.
+
+   Additionally, the option `--read-all-packs` has been added, implementing the previous
+   behavior.
+
+   https://github.com/restic/restic/pull/3006
+   https://github.com/restic/restic/issue/2547
+
+ * Enhancement #3048: Add more checks for index and pack files in the `check` command
+
+   The `check` command run with the `--read-data` or `--read-data-subset` options used to only
+   verify only the pack file content - it did not check if the blobs within the pack are correctly
+   contained in the index.
+
+   A check for the latter is now in place, which can print the following error:
+
+   Blob ID is not contained in index or position is incorrect
+
+   Another test is also added, which compares pack file sizes computed from the index and the pack
+   header with the actual file size. This test is able to detect truncated pack files.
+
+   If the index is not correct, it can be rebuilt by using the `rebuild-index` command.
+
+   Having added these tests, `restic check` is now able to detect non-existing blobs which are
+   wrongly referenced in the index. This situation could have lead to missing data.
+
+   https://github.com/restic/restic/pull/3048
+   https://github.com/restic/restic/pull/3082
+
+ * Enhancement #2433: Make the `dump` command support `zip` format
+
+   Previously, restic could dump the contents of a whole folder structure only in the `tar`
+   format. The `dump` command now has a new flag to change output format to `zip`. Just pass
+   `--archive zip` as an option to `restic dump`.
+
+   https://github.com/restic/restic/pull/2433
+   https://github.com/restic/restic/pull/3081
+
+ * Enhancement #3099: Reduce memory usage of `check` command
+
+   The `check` command now requires less memory if it is run without the `--check-unused` option.
+
+   https://github.com/restic/restic/pull/3099
+
+ * Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune`
+
+   The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find
+   used data. This snapshot traversal is now parallized which can speed up this step several
+   times.
+
+   In addition the `check` command now reports how many snapshots have already been processed.
+
+   https://github.com/restic/restic/pull/3106
+
+ * Enhancement #3130: Parallelize reading of locks and snapshots
+
+   Restic used to read snapshots sequentially. For repositories containing many snapshots this
+   slowed down commands which have to read all snapshots.
+
+   Now the reading of snapshots is parallelized. This speeds up for example `prune`, `backup` and
+   other commands that search for snapshots with certain properties or which have to find the
+   `latest` snapshot.
+
+   The speed up also applies to locks stored in the backup repository.
+
+   https://github.com/restic/restic/pull/3130
+   https://github.com/restic/restic/pull/3174
+
+ * Enhancement #3254: Enable HTTP/2 for backend connections
+
+   Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 depending on
+   what the server supports. But for compatibility this mechanism is disabled if DialContext is
+   used (which is the case for restic). This change allows restic's HTTP client to negotiate
+   HTTP/2 if supported by the server.
+
+   https://github.com/restic/restic/pull/3254
+
+
+Changelog for restic 0.11.0 (2020-11-05)
+=======================================
+
+The following sections list the changes in restic 0.11.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1212: Restore timestamps and permissions on intermediate directories
+ * Fix #1756: Mark repository files as read-only when using the local backend
+ * Fix #2241: Hide password in REST backend repository URLs
+ * Fix #2319: Correctly dump directories into tar files
+ * Fix #2491: Don't require `self-update --output` placeholder file
+ * Fix #2834: Fix rare cases of backup command hanging forever
+ * Fix #2938: Fix manpage formatting
+ * Fix #2942: Make --exclude-larger-than handle disappearing files
+ * Fix #2951: Restic generate, help and self-update no longer check passwords
+ * Fix #2979: Make snapshots --json output [] instead of null when no snapshots
+ * Enh #2969: Optimize check for unchanged files during backup
+ * Enh #340: Add support for Volume Shadow Copy Service (VSS) on Windows
+ * Enh #2849: Authenticate to Google Cloud Storage with access token
+ * Enh #1458: New option --repository-file
+ * Enh #2978: Warn if parent snapshot cannot be loaded during backup
+
+Details
+-------
+
+ * Bugfix #1212: Restore timestamps and permissions on intermediate directories
+
+   When using the `--include` option of the restore command, restic restored timestamps and
+   permissions only on directories selected by the include pattern. Intermediate directories,
+   which are necessary to restore files located in sub- directories, were created with default
+   permissions. We've fixed the restore command to restore timestamps and permissions for these
+   directories as well.
+
+   https://github.com/restic/restic/issues/1212
+   https://github.com/restic/restic/issues/1402
+   https://github.com/restic/restic/pull/2906
+
+ * Bugfix #1756: Mark repository files as read-only when using the local backend
+
+   Files stored in a local repository were marked as writeable on the filesystem for non-Windows
+   systems, which did not prevent accidental file modifications outside of restic. In addition,
+   the local backend did not work with certain filesystems and network mounts which do not permit
+   modifications of file permissions.
+
+   Restic now marks files stored in a local repository as read-only on the filesystem on
+   non-Windows systems. The error handling is improved to support more filesystems.
+
+   https://github.com/restic/restic/issues/1756
+   https://github.com/restic/restic/issues/2157
+   https://github.com/restic/restic/pull/2989
+
+ * Bugfix #2241: Hide password in REST backend repository URLs
+
+   When using a password in the REST backend repository URL, the password could in some cases be
+   included in the output from restic, e.g. when initializing a repo or during an error.
+
+   The password is now replaced with "***" where applicable.
+
+   https://github.com/restic/restic/issues/2241
+   https://github.com/restic/restic/pull/2658
+
+ * Bugfix #2319: Correctly dump directories into tar files
+
+   The dump command previously wrote directories in a tar file in a way which can cause
+   compatibility problems. This caused, for example, 7zip on Windows to not open tar files
+   containing directories. In addition it was not possible to dump directories with extended
+   attributes. These compatibility problems are now corrected.
+
+   In addition, a tar file now includes the name of the owner and group of a file.
+
+   https://github.com/restic/restic/issues/2319
+   https://github.com/restic/restic/pull/3039
+
+ * Bugfix #2491: Don't require `self-update --output` placeholder file
+
+   `restic self-update --output /path/to/new-restic` used to require that new-restic was an
+   existing file, to be overwritten. Now it's possible to download an updated restic binary to a
+   new path, without first having to create a placeholder file.
+
+   https://github.com/restic/restic/issues/2491
+   https://github.com/restic/restic/pull/2937
+
+ * Bugfix #2834: Fix rare cases of backup command hanging forever
+
+   We've fixed an issue with the backup progress reporting which could cause restic to hang
+   forever right before finishing a backup.
+
+   https://github.com/restic/restic/issues/2834
+   https://github.com/restic/restic/pull/2963
+
+ * Bugfix #2938: Fix manpage formatting
+
+   The manpage formatting in restic v0.10.0 was garbled, which is fixed now.
+
+   https://github.com/restic/restic/issues/2938
+   https://github.com/restic/restic/pull/2977
+
+ * Bugfix #2942: Make --exclude-larger-than handle disappearing files
+
+   There was a small bug in the backup command's --exclude-larger-than option where files that
+   disappeared between scanning and actually backing them up to the repository caused a panic.
+   This is now fixed.
+
+   https://github.com/restic/restic/issues/2942
+
+ * Bugfix #2951: Restic generate, help and self-update no longer check passwords
+
+   The commands `restic cache`, `generate`, `help` and `self-update` don't need passwords, but
+   they previously did run the RESTIC_PASSWORD_COMMAND (if set in the environment), prompting
+   users to authenticate for no reason. They now skip running the password command.
+
+   https://github.com/restic/restic/issues/2951
+   https://github.com/restic/restic/pull/2987
+
+ * Bugfix #2979: Make snapshots --json output [] instead of null when no snapshots
+
+   Restic previously output `null` instead of `[]` for the `--json snapshots` command, when
+   there were no snapshots in the repository. This caused some minor problems when parsing the
+   output, but is now fixed such that `[]` is output when the list of snapshots is empty.
+
+   https://github.com/restic/restic/issues/2979
+   https://github.com/restic/restic/pull/2984
+
+ * Enhancement #2969: Optimize check for unchanged files during backup
+
+   During a backup restic skips processing files which have not changed since the last backup run.
+   Previously this required opening each file once which can be slow on network filesystems. The
+   backup command now checks for file changes before opening a file. This considerably reduces
+   the time to create a backup on network filesystems.
+
+   https://github.com/restic/restic/issues/2969
+   https://github.com/restic/restic/pull/2970
+
+ * Enhancement #340: Add support for Volume Shadow Copy Service (VSS) on Windows
+
+   Volume Shadow Copy Service allows read access to files that are locked by another process using
+   an exclusive lock through a filesystem snapshot. Restic was unable to backup those files
+   before. This update enables backing up these files.
+
+   This needs to be enabled explicitely using the --use-fs-snapshot option of the backup
+   command.
+
+   https://github.com/restic/restic/issues/340
+   https://github.com/restic/restic/pull/2274
+
+ * Enhancement #2849: Authenticate to Google Cloud Storage with access token
+
+   When using the GCS backend, it is now possible to authenticate with OAuth2 access tokens
+   instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN environment variable.
+
+   https://github.com/restic/restic/pull/2849
+
+ * Enhancement #1458: New option --repository-file
+
+   We've added a new command-line option --repository-file as an alternative to -r. This allows
+   to read the repository URL from a file in order to prevent certain types of information leaks,
+   especially for URLs containing credentials.
+
+   https://github.com/restic/restic/issues/1458
+   https://github.com/restic/restic/issues/2900
+   https://github.com/restic/restic/pull/2910
+
+ * Enhancement #2978: Warn if parent snapshot cannot be loaded during backup
+
+   During a backup restic uses the parent snapshot to check whether a file was changed and has to be
+   backed up again. For this check the backup has to read the directories contained in the old
+   snapshot. If a tree blob cannot be loaded, restic now warns about this problem with the backup
+   repository.
+
+   https://github.com/restic/restic/pull/2978
+
+
 Changelog for restic 0.10.0 (2020-09-19)
 =======================================
 

CONTRIBUTING.md

@@ -16,7 +16,8 @@ help also.
 
 The restic project uses the GitHub infrastructure (see the
 [project page](https://github.com/restic/restic)) for all related discussions
-as well as the `#restic` channel on `irc.freenode.net`.
+as well as the [forum](https://forum.restic.net/) and the `#restic` channel
+on [irc.freenode.net](https://kiwiirc.com/nextclient/irc.freenode.net/restic).
 
 If you want to find an area that currently needs improving have a look at the
 open issues listed at the
@@ -25,7 +26,10 @@ for discussing enhancement to the restic tools.
 
 If you are unsure what to do, please have a look at the issues, especially
 those tagged
-[minor complexity](https://github.com/restic/restic/labels/minor%20complexity).
+[minor complexity](https://github.com/restic/restic/labels/help%3A%20minor%20complexity)
+or [good first issue](https://github.com/restic/restic/labels/help%3A%20good%20first%20issue).
+If you are already a bit experienced with the restic internals, take a look
+at the issues tagged as [help wanted](https://github.com/restic/restic/labels/help%3A%20wanted).
 
 
 Reporting Bugs
@@ -63,7 +67,7 @@ Development Environment
 The repository contains the code written for restic in the directories
 `cmd/` and `internal/`.
 
-Restic requires Go version 1.12 or later for compiling. Clone the repo (without
+Restic requires Go version 1.13 or later for compiling. Clone the repo (without
 having `$GOPATH` set) and `cd` into the directory:
 
     $ unset GOPATH
@@ -74,7 +78,7 @@ Then use the `go` tool to build restic:
 
     $ go build ./cmd/restic
     $ ./restic version
-    restic 0.9.6-dev (compiled manually) compiled with go1.14 on linux/amd64
+    restic 0.10.0-dev (compiled manually) compiled with go1.15.2 on linux/amd64
 
 You can run all tests with the following command:
 
@@ -137,6 +141,14 @@ Installing the script `fmt-check` from https://github.com/edsrzf/gofmt-git-hook
 locally as a pre-commit hook checks formatting before committing automatically,
 just copy this script to `.git/hooks/pre-commit`.
 
+The project is using the program
+[`golangci-lint`](https://github.com/golangci/golangci-lint) to run a list of
+linters and checkers. It will be run on the code when you submit a PR. In order
+to check your code beforehand, you can run `golangci-lint run` manually.
+Eventually, we will enable `golangci-lint` for the whole code base. For now,
+you can ignore warnings printed for lines you did not modify, those will be
+ignored by the CI.
+
 For each pull request, several different systems run the integration tests on
 Linux, macOS and Windows. We won't merge any code that does not pass all tests
 for all systems, so when a tests fails, try to find out what's wrong and fix

README.md

@@ -0,0 +1,113 @@
+[![Documentation](https://readthedocs.org/projects/restic/badge/?version=latest)](https://restic.readthedocs.io/en/latest/?badge=latest)
+[![Build Status](https://github.com/restic/restic/workflows/test/badge.svg)](https://github.com/restic/restic/actions?query=workflow%3Atest)
+[![Go Report Card](https://goreportcard.com/badge/github.com/restic/restic)](https://goreportcard.com/report/github.com/restic/restic)
+
+# Introduction
+
+restic is a backup program that is fast, efficient and secure. It supports the three major operating systems (Linux, macOS, Windows) and a few smaller ones (FreeBSD, OpenBSD).
+
+For detailed usage and installation instructions check out the [documentation](https://restic.readthedocs.io/en/latest).
+
+You can ask questions in our [Discourse forum](https://forum.restic.net).
+
+Quick start
+-----------
+
+Once you've [installed](https://restic.readthedocs.io/en/latest/020_installation.html) restic, start
+off with creating a repository for your backups:
+
+    $ restic init --repo /tmp/backup
+    enter password for new backend:
+    enter password again:
+    created restic backend 085b3c76b9 at /tmp/backup
+    Please note that knowledge of your password is required to access the repository.
+    Losing your password means that your data is irrecoverably lost.
+
+and add some data:
+
+    $ restic --repo /tmp/backup backup ~/work
+    enter password for repository:
+    scan [/home/user/work]
+    scanned 764 directories, 1816 files in 0:00
+    [0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
+    duration: 0:29, 54.47MiB/s
+    snapshot 40dc1520 saved
+
+Next you can either use `restic restore` to restore files or use `restic
+mount` to mount the repository via fuse and browse the files from previous
+snapshots.
+
+For more options check out the [online documentation](https://restic.readthedocs.io/en/latest/).
+
+# Backends
+
+Saving a backup on the same machine is nice but not a real backup strategy.
+Therefore, restic supports the following backends for storing backups natively:
+
+- [Local directory](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local)
+- [sftp server (via SSH)](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp)
+- [HTTP REST server](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) ([protocol](doc/100_references.rst#rest-backend), [rest-server](https://github.com/restic/rest-server))
+- [AWS S3](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) (either from Amazon or using the [Minio](https://minio.io) server)
+- [OpenStack Swift](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift)
+- [BackBlaze B2](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2)
+- [Microsoft Azure Blob Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage)
+- [Google Cloud Storage](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage)
+- And many other services via the [rclone](https://rclone.org) [Backend](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone)
+
+# Design Principles
+
+Restic is a program that does backups right and was designed with the
+following principles in mind:
+
+-  **Easy:** Doing backups should be a frictionless process, otherwise
+   you might be tempted to skip it. Restic should be easy to configure
+   and use, so that, in the event of a data loss, you can just restore
+   it. Likewise, restoring data should not be complicated.
+
+-  **Fast**: Backing up your data with restic should only be limited by
+   your network or hard disk bandwidth so that you can backup your files
+   every day. Nobody does backups if it takes too much time. Restoring
+   backups should only transfer data that is needed for the files that
+   are to be restored, so that this process is also fast.
+
+-  **Verifiable**: Much more important than backup is restore, so restic
+   enables you to easily verify that all data can be restored.
+
+-  **Secure**: Restic uses cryptography to guarantee confidentiality and
+   integrity of your data. The location the backup data is stored is
+   assumed not to be a trusted environment (e.g. a shared space where
+   others like system administrators are able to access your backups).
+   Restic is built to secure your data against such attackers.
+
+-  **Efficient**: With the growth of data, additional snapshots should
+   only take the storage of the actual increment. Even more, duplicate
+   data should be de-duplicated before it is actually written to the
+   storage back end to save precious backup space.
+
+# Reproducible Builds
+
+The binaries released with each restic version starting at 0.6.1 are
+[reproducible](https://reproducible-builds.org/), which means that you can
+reproduce a byte identical version from the source code for that
+release. Instructions on how to do that are contained in the
+[builder repository](https://github.com/restic/builder).
+
+News
+----
+
+You can follow the restic project on Twitter [@resticbackup](https://twitter.com/resticbackup) or by subscribing to
+the [project blog](https://restic.net/blog/).
+
+License
+-------
+
+Restic is licensed under [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause). You can find the
+complete text in [``LICENSE``](LICENSE).
+
+Sponsorship
+-----------
+
+Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
+Storage are sponsored by [AppsCode](https://appscode.com)!
+
+[![Sponsored by AppsCode](https://cdn.appscode.com/images/logo/appscode/ac-logo-color.png)](https://appscode.com)

README.rst

@@ -1,135 +0,0 @@
-|Documentation| |Build Status| |Build status| |Report Card| |Say Thanks| |Reviewed by Hound|
-
-Introduction
-------------
-
-restic is a backup program that is fast, efficient and secure. It supports the three major operating systems (Linux, macOS, Windows) and a few smaller ones (FreeBSD, OpenBSD).
-
-For detailed usage and installation instructions check out the `documentation <https://restic.readthedocs.io/en/latest>`__.
-
-You can ask questions in our `Discourse forum <https://forum.restic.net>`__.
-
-Quick start
------------
-
-Once you've `installed
-<https://restic.readthedocs.io/en/latest/020_installation.html>`__ restic, start
-off with creating a repository for your backups:
-
-.. code-block:: console
-
-    $ restic init --repo /tmp/backup
-    enter password for new backend:
-    enter password again:
-    created restic backend 085b3c76b9 at /tmp/backup
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
-
-and add some data:
-
-.. code-block:: console
-
-    $ restic --repo /tmp/backup backup ~/work
-    enter password for repository:
-    scan [/home/user/work]
-    scanned 764 directories, 1816 files in 0:00
-    [0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
-    duration: 0:29, 54.47MiB/s
-    snapshot 40dc1520 saved
-
-Next you can either use ``restic restore`` to restore files or use ``restic
-mount`` to mount the repository via fuse and browse the files from previous
-snapshots.
-
-For more options check out the `online documentation <https://restic.readthedocs.io/en/latest/>`__.
-
-Backends
---------
-
-Saving a backup on the same machine is nice but not a real backup strategy.
-Therefore, restic supports the following backends for storing backups natively:
-
-- `Local directory <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#local>`__
-- `sftp server (via SSH) <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#sftp>`__
-- `HTTP REST server <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server>`__ (`protocol <doc/100_references.rst#rest-backend>`__ `rest-server <https://github.com/restic/rest-server>`__)
-- `AWS S3 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3>`__ (either from Amazon or using the `Minio <https://minio.io>`__ server)
-- `OpenStack Swift <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#openstack-swift>`__
-- `BackBlaze B2 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2>`__
-- `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage>`__
-- `Google Cloud Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage>`__
-- And many other services via the `rclone <https://rclone.org>`__ `Backend <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone>`__
-
-Design Principles
------------------
-
-Restic is a program that does backups right and was designed with the
-following principles in mind:
-
--  **Easy:** Doing backups should be a frictionless process, otherwise
-   you might be tempted to skip it. Restic should be easy to configure
-   and use, so that, in the event of a data loss, you can just restore
-   it. Likewise, restoring data should not be complicated.
-
--  **Fast**: Backing up your data with restic should only be limited by
-   your network or hard disk bandwidth so that you can backup your files
-   every day. Nobody does backups if it takes too much time. Restoring
-   backups should only transfer data that is needed for the files that
-   are to be restored, so that this process is also fast.
-
--  **Verifiable**: Much more important than backup is restore, so restic
-   enables you to easily verify that all data can be restored.
-
--  **Secure**: Restic uses cryptography to guarantee confidentiality and
-   integrity of your data. The location the backup data is stored is
-   assumed not to be a trusted environment (e.g. a shared space where
-   others like system administrators are able to access your backups).
-   Restic is built to secure your data against such attackers.
-
--  **Efficient**: With the growth of data, additional snapshots should
-   only take the storage of the actual increment. Even more, duplicate
-   data should be de-duplicated before it is actually written to the
-   storage back end to save precious backup space.
-
-Reproducible Builds
--------------------
-
-The binaries released with each restic version starting at 0.6.1 are
-`reproducible <https://reproducible-builds.org/>`__, which means that you can
-easily reproduce a byte identical version from the source code for that
-release. Instructions on how to do that are contained in the
-`builder repository <https://github.com/restic/builder>`__.
-
-News
-----
-
-You can follow the restic project on Twitter `@resticbackup <https://twitter.com/resticbackup>`__ or by subscribing to
-the `development blog <https://restic.net/blog/>`__.
-
-License
--------
-
-Restic is licensed under `BSD 2-Clause License <https://opensource.org/licenses/BSD-2-Clause>`__. You can find the
-complete text in ``LICENSE``.
-
-Sponsorship
------------
-
-Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
-Storage are sponsored by `AppsCode <https://appscode.com>`__!
-
-|AppsCode|
-
-.. |Documentation| image:: https://readthedocs.org/projects/restic/badge/?version=latest
-   :target: https://restic.readthedocs.io/en/latest/?badge=latest
-.. |Build Status| image:: https://travis-ci.com/restic/restic.svg?branch=master
-   :target: https://travis-ci.com/restic/restic
-.. |Build status| image:: https://ci.appveyor.com/api/projects/status/nuy4lfbgfbytw92q/branch/master?svg=true
-   :target: https://ci.appveyor.com/project/fd0/restic/branch/master
-.. |Report Card| image:: https://goreportcard.com/badge/github.com/restic/restic
-   :target: https://goreportcard.com/report/github.com/restic/restic
-.. |Say Thanks| image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg
-   :target: https://saythanks.io/to/restic
-.. |AppsCode| image:: https://cdn.appscode.com/images/logo/appscode/ac-logo-color.png
-   :target: https://appscode.com
-.. |Reviewed by Hound| image:: https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg
-   :target: https://houndci.com

VERSION

@@ -1 +1 @@
-0.10.0
+0.12.0

appveyor.yml

@@ -1,32 +0,0 @@
-clone_folder: c:\restic
-
-environment:
-  GOPATH: c:\gopath
-
-branches:
-  only:
-    - master
-
-cache:
-  - '%LocalAppData%\go-build'
-
-init:
-  - ps: >-
-      $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
-
-      if ($app) {
-        $app.Uninstall()
-      }
-
-install:
-  - rmdir c:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.15.2.windows-amd64.msi
-  - msiexec /i go1.15.2.windows-amd64.msi /q
-  - go version
-  - go env
-  - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
-  - 7z x tar.zip bin/tar.exe
-  - set PATH=bin/;%PATH%
-
-build_script:
-  - go run run_integration_tests.go

changelog/0.11.0_2020-11-05/issue-1212

@@ -0,0 +1,11 @@
+Bugfix: Restore timestamps and permissions on intermediate directories
+
+When using the `--include` option of the restore command, restic restored
+timestamps and permissions only on directories selected by the include pattern.
+Intermediate directories, which are necessary to restore files located in sub-
+directories, were created with default permissions. We've fixed the restore
+command to restore timestamps and permissions for these directories as well.
+
+https://github.com/restic/restic/issues/1212
+https://github.com/restic/restic/issues/1402
+https://github.com/restic/restic/pull/2906

changelog/0.11.0_2020-11-05/issue-1756

@@ -0,0 +1,15 @@
+Bugfix: Mark repository files as read-only when using the local backend
+
+Files stored in a local repository were marked as writeable on the
+filesystem for non-Windows systems, which did not prevent accidental file
+modifications outside of restic. In addition, the local backend did not work
+with certain filesystems and network mounts which do not permit modifications
+of file permissions.
+
+restic now marks files stored in a local repository as read-only on the
+filesystem on non-Windows systems. The error handling is improved to support
+more filesystems.
+
+https://github.com/restic/restic/issues/1756
+https://github.com/restic/restic/issues/2157
+https://github.com/restic/restic/pull/2989

changelog/0.11.0_2020-11-05/issue-2241

@@ -0,0 +1,10 @@
+Bugfix: Hide password in REST backend repository URLs
+
+When using a password in the REST backend repository URL,
+the password could in some cases be included in the output
+from restic, e.g. when initializing a repo or during an error.
+
+The password is now replaced with "***" where applicable.
+
+https://github.com/restic/restic/issues/2241
+https://github.com/restic/restic/pull/2658

changelog/0.11.0_2020-11-05/issue-2319

@@ -0,0 +1,12 @@
+Bugfix: Correctly dump directories into tar files
+
+The dump command previously wrote directories in a tar file in a way which
+can cause compatibility problems. This caused, for example, 7zip on Windows
+to not open tar files containing directories. In addition it was not possible
+to dump directories with extended attributes. These compatibility problems
+are now corrected.
+
+In addition, a tar file now includes the name of the owner and group of a file.
+
+https://github.com/restic/restic/issues/2319
+https://github.com/restic/restic/pull/3039

changelog/0.11.0_2020-11-05/issue-2491

@@ -0,0 +1,9 @@
+Bugfix: Don't require `self-update --output` placeholder file
+
+`restic self-update --output /path/to/new-restic` used to require that
+new-restic was an existing file, to be overwritten. Now it's possible
+to download an updated restic binary to a new path, without first
+having to create a placeholder file.
+
+https://github.com/restic/restic/issues/2491
+https://github.com/restic/restic/pull/2937

changelog/0.11.0_2020-11-05/issue-2834

@@ -0,0 +1,7 @@
+Bugfix: Fix rare cases of backup command hanging forever
+
+We've fixed an issue with the backup progress reporting which could cause
+restic to hang forever right before finishing a backup.
+
+https://github.com/restic/restic/issues/2834
+https://github.com/restic/restic/pull/2963

changelog/0.11.0_2020-11-05/issue-2938

@@ -0,0 +1,6 @@
+Bugfix: Fix manpage formatting
+
+The manpage formatting in restic v0.10.0 was garbled, which is fixed now.
+
+https://github.com/restic/restic/issues/2938
+https://github.com/restic/restic/pull/2977

changelog/0.11.0_2020-11-05/issue-2942

@@ -0,0 +1,7 @@
+Bugfix: Make --exclude-larger-than handle disappearing files
+
+There was a small bug in the backup command's --exclude-larger-than
+option where files that disappeared between scanning and actually
+backing them up to the repository caused a panic. This is now fixed.
+
+https://github.com/restic/restic/issues/2942

changelog/0.11.0_2020-11-05/issue-2951

@@ -0,0 +1,9 @@
+Bugfix: restic generate, help and self-update no longer check passwords
+
+The commands `restic cache`, `generate`, `help` and `self-update` don't need
+passwords, but they previously did run the RESTIC_PASSWORD_COMMAND (if set in
+the environment), prompting users to authenticate for no reason. They now skip
+running the password command.
+
+https://github.com/restic/restic/issues/2951
+https://github.com/restic/restic/pull/2987

changelog/0.11.0_2020-11-05/issue-2969

@@ -0,0 +1,9 @@
+Enhancement: Optimize check for unchanged files during backup
+
+During a backup restic skips processing files which have not changed since the last backup run.
+Previously this required opening each file once which can be slow on network filesystems. The
+backup command now checks for file changes before opening a file. This considerably reduces
+the time to create a backup on network filesystems.
+
+https://github.com/restic/restic/issues/2969
+https://github.com/restic/restic/pull/2970

changelog/0.11.0_2020-11-05/issue-2979

@@ -0,0 +1,9 @@
+Bugfix: Make snapshots --json output [] instead of null when no snapshots
+
+Restic previously output `null` instead of `[]` for the `--json snapshots`
+command, when there were no snapshots in the repository. This caused some
+minor problems when parsing the output, but is now fixed such that `[]` is
+output when the list of snapshots is empty.
+
+https://github.com/restic/restic/issues/2979
+https://github.com/restic/restic/pull/2984

changelog/0.11.0_2020-11-05/issue-340

@@ -0,0 +1,12 @@
+Enhancement: Add support for Volume Shadow Copy Service (VSS) on Windows
+
+Volume Shadow Copy Service allows read access to files that are locked by
+another process using an exclusive lock through a filesystem snapshot. Restic
+was unable to backup those files before. This update enables backing up these
+files.
+
+This needs to be enabled explicitely using the --use-fs-snapshot option of the
+backup command.
+
+https://github.com/restic/restic/issues/340
+https://github.com/restic/restic/pull/2274

changelog/0.11.0_2020-11-05/pull-2849

@@ -0,0 +1,7 @@
+Enhancement: Authenticate to Google Cloud Storage with access token
+
+When using the GCS backend, it is now possible to authenticate with OAuth2
+access tokens instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN
+environment variable.
+
+https://github.com/restic/restic/pull/2849

changelog/0.11.0_2020-11-05/pull-2910

@@ -0,0 +1,10 @@
+Enhancement: New option --repository-file
+
+We've added a new command-line option --repository-file as an alternative
+to -r. This allows to read the repository URL from a file in order to
+prevent certain types of information leaks, especially for URLs containing
+credentials.
+
+https://github.com/restic/restic/issues/1458
+https://github.com/restic/restic/issues/2900
+https://github.com/restic/restic/pull/2910

changelog/0.11.0_2020-11-05/pull-2978

@@ -0,0 +1,8 @@
+Enhancement: Warn if parent snapshot cannot be loaded during backup
+
+During a backup restic uses the parent snapshot to check whether a file was
+changed and has to be backed up again. For this check the backup has to read
+the directories contained in the old snapshot. If a tree blob cannot be
+loaded, restic now warns about this problem with the backup repository.
+
+https://github.com/restic/restic/pull/2978

changelog/0.12.0_2021-02-14/issue-1681

@@ -0,0 +1,10 @@
+Bugfix: Make `mount` not create missing mount point directory
+
+When specifying a non-existent directory as mount point for the `mount`
+command, restic used to create the specified directory automatically.
+
+This has now changed such that restic instead gives an error when the
+specified directory for the mount point does not exist.
+
+https://github.com/restic/restic/issues/1681
+https://github.com/restic/restic/pull/3008

changelog/0.12.0_2021-02-14/issue-1800

@@ -0,0 +1,8 @@
+Bugfix: Ignore `no data available` filesystem error during backup
+
+Restic was unable to backup files on some filesystems, for example certain
+configurations of CIFS on Linux which return a `no data available` error
+when reading extended attributes. These errors are now ignored.
+
+https://github.com/restic/restic/issues/1800
+https://github.com/restic/restic/pull/3034

changelog/0.12.0_2021-02-14/issue-2186

@@ -0,0 +1,8 @@
+Enhancement: Allow specifying percentage in `check --read-data-subset`
+
+We've enhanced the `check` command's `--read-data-subset` option to also accept
+a percentage (e.g. `2.5%` or `10%`). This will check the given percentage of
+pack files (which are randomly selected on each run).
+
+https://github.com/restic/restic/issues/2186
+https://github.com/restic/restic/pull/3038

changelog/0.12.0_2021-02-14/issue-2453

@@ -0,0 +1,14 @@
+Enhancement: Report permanent/fatal backend errors earlier
+
+When encountering errors in reading from or writing to storage backends,
+restic retries the failing operation up to nine times (for a total of ten
+attempts). It used to retry all backend operations, but now detects some
+permanent error conditions so that it can report fatal errors earlier.
+
+Permanent failures include local disks being full, SSH connections
+dropping and permission errors.
+
+https://github.com/restic/restic/issues/2453
+https://github.com/restic/restic/pull/3170
+https://github.com/restic/restic/issues/3180
+https://github.com/restic/restic/pull/3181

changelog/0.12.0_2021-02-14/issue-2528

@@ -0,0 +1,21 @@
+Enhancement: Add Alibaba/Aliyun OSS support in the `s3` backend
+
+A new extended option `s3.bucket-lookup` has been added to support
+Alibaba/Aliyun OSS in the `s3` backend. The option can be set to one
+of the following values:
+
+- `auto` - Existing behaviour
+- `dns` - Use DNS style bucket access
+- `path` - Use path style bucket access
+
+To make the `s3` backend work with Alibaba/Aliyun OSS you must set
+`s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For
+example:
+
+    restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init
+
+Note that `s3.region` must be set, otherwise the MinIO SDK tries to
+look it up and it seems that Alibaba doesn't support that properly.
+
+https://github.com/restic/restic/issues/2528
+https://github.com/restic/restic/pull/2535

changelog/0.12.0_2021-02-14/issue-2563

@@ -0,0 +1,9 @@
+Bugfix: Report the correct owner of directories in FUSE mounts
+
+Restic 0.10.0 changed the FUSE mount to always report the current user
+as the owner of directories within the FUSE mount, which is incorrect.
+
+This is now changed back to reporting the correct owner of a directory.
+
+https://github.com/restic/restic/issues/2563
+https://github.com/restic/restic/pull/3141

changelog/0.12.0_2021-02-14/issue-2688

@@ -0,0 +1,31 @@
+Bugfix: Make `backup` and `tag` commands separate tags by comma
+
+Running `restic backup --tag foo,bar` previously created snapshots with one
+single tag containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`).
+
+Similarly, the `tag` command's `--set`, `--add` and `--remove` options would
+treat `foo,bar` as one tag instead of two tags. This was inconsistent with
+other commands and often unexpected when one intended `foo,bar` to mean two
+tags.
+
+To be consistent in all commands, restic now interprets `foo,bar` to mean two
+separate tags (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere,
+including in the `backup` and `tag` commands.
+
+NOTE: This change might result in unexpected behavior in cases where you use
+the `forget` command and filter on tags like `foo,bar`. Snapshots previously
+backed up with `--tag foo,bar` will still not match that filter, but snapshots
+saved from now on will match that filter.
+
+To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can
+first generate a list of the relevant snapshots using a command like:
+
+    restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id'
+
+and then use `restic tag --set foo --set bar snapshotID [...]` to set the new
+tags. Please adjust the commands to include real tag names and any additional
+tags, as well as the list of snapshots to process.
+
+https://github.com/restic/restic/issues/2688
+https://github.com/restic/restic/pull/2690
+https://github.com/restic/restic/pull/3197

changelog/0.12.0_2021-02-14/issue-2706

@@ -0,0 +1,17 @@
+Enhancement: Configurable progress reports for non-interactive terminals
+
+The `backup`, `check` and `prune` commands never printed any progress
+reports on non-interactive terminals. This behavior is now configurable
+using the `RESTIC_PROGRESS_FPS` environment variable. Use for example a
+value of `1` for an update every second, or `0.01666` for an update every
+minute.
+
+The `backup` command now also prints the current progress when restic
+receives a `SIGUSR1` signal.
+
+Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1`
+signal prints a status report even when `--quiet` was specified.
+
+https://github.com/restic/restic/issues/2706
+https://github.com/restic/restic/issues/3194
+https://github.com/restic/restic/pull/3199

changelog/0.12.0_2021-02-14/issue-2739

@@ -0,0 +1,5 @@
+Bugfix: Make the `cat` command respect the `--no-lock` option
+
+The `cat` command would not respect the `--no-lock` flag. This is now fixed.
+
+https://github.com/restic/restic/issues/2739

changelog/0.12.0_2021-02-14/issue-2944

@@ -0,0 +1,13 @@
+Enhancement: Add `backup` options `--files-from-{verbatim,raw}`
+
+The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a
+list of files to back up from a file. Unlike the existing `--files-from`
+option, these options do not interpret the listed filenames as glob patterns;
+instead, whitespace in filenames is preserved as-is and no pattern expansion is
+done. Please see the documentation for specifics.
+
+These new options are highly recommended over `--files-from`, when using a
+script to generate the list of files to back up.
+
+https://github.com/restic/restic/issues/2944
+https://github.com/restic/restic/issues/3013

changelog/0.12.0_2021-02-14/issue-3083

@@ -0,0 +1,18 @@
+Enhancement: Allow usage of deprecated S3 `ListObjects` API
+
+Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken
+`ListObjectsV2` implementation which causes problems for restic when using
+their API endpoints. When a broken server implementation is used, restic prints
+errors similar to the following:
+
+    List() returned error: Truncated response should have continuation token set
+
+As a temporary workaround, restic now allows using the older `ListObjects`
+endpoint by setting the `s3.list-objects-v1` extended option, for instance:
+
+    restic -o s3.list-objects-v1=true snapshots
+
+Please note that this option may be removed in future versions of restic.
+
+https://github.com/restic/restic/issues/3083
+https://github.com/restic/restic/pull/3085

changelog/0.12.0_2021-02-14/issue-3090

@@ -0,0 +1,10 @@
+Bugfix: The `--use-fs-snapshot` option now works on windows/386
+
+Restic failed to create VSS snapshots on windows/386 with the following error:
+
+    GetSnapshotProperties() failed: E_INVALIDARG (0x80070057)
+
+This is now fixed.
+
+https://github.com/restic/restic/issues/3087
+https://github.com/restic/restic/pull/3090

changelog/0.12.0_2021-02-14/issue-3095

@@ -0,0 +1,17 @@
+Change: Deleting files on Google Drive now moves them to the trash
+
+When deleting files on Google Drive via the `rclone` backend, restic used to
+bypass the trash folder required that one used the `-o rclone.args` option to
+enable usage of the trash folder. This ensured that deleted files in Google
+Drive were not kept indefinitely in the trash folder. However, since Google
+Drive's trash retention policy changed to deleting trashed files after 30 days,
+this is no longer needed.
+
+Restic now leaves it up to rclone and its configuration to use or not use the
+trash folder when deleting files. The default is to use the trash folder, as
+of rclone 1.53.2. To re-enable the restic 0.11 behavior, set the
+`RCLONE_DRIVE_USE_TRASH` environment variable or change the rclone
+configuration. See the rclone documentation for more details.
+
+https://github.com/restic/restic/issues/3095
+https://github.com/restic/restic/pull/3102

changelog/0.12.0_2021-02-14/issue-3100

@@ -0,0 +1,10 @@
+Bugfix: Do not require gs bucket permissions when running `init`
+
+Restic used to require bucket level permissions for the `gs` backend 
+in order to initialize a restic repository.
+
+It now allows a `gs` service account to initialize a repository if the
+bucket does exist and the service account has permissions to write/read
+to that bucket.
+
+https://github.com/restic/restic/issues/3100

changelog/0.12.0_2021-02-14/issue-3111

@@ -0,0 +1,9 @@
+Bugfix: Correctly detect output redirection for `backup` command on Windows
+
+On Windows, since restic 0.10.0 the `backup` command did not properly detect
+when the output was redirected to a file. This caused restic to output
+terminal control characters. This has been fixed by correcting the terminal
+detection.
+
+https://github.com/restic/restic/issues/3111
+https://github.com/restic/restic/pull/3150

changelog/0.12.0_2021-02-14/issue-3147

@@ -0,0 +1,11 @@
+Enhancement: Support additional environment variables for Swift authentication
+
+The `swift` backend now supports the following additional environment variables
+for passing authentication details to restic:
+`OS_USER_ID`, `OS_USER_DOMAIN_ID`, `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
+
+Depending on the `openrc` configuration file these might be required when the
+user and project domains differ from one another.
+
+https://github.com/restic/restic/issues/3147
+https://github.com/restic/restic/pull/3158

changelog/0.12.0_2021-02-14/issue-3151

@@ -0,0 +1,9 @@
+Bugfix: Don't create invalid snapshots when `backup` is interrupted
+
+When canceling a backup run at a certain moment it was possible that
+restic created a snapshot with an invalid "null" tree. This caused
+`check` and other operations to fail. The `backup` command now properly
+handles interruptions and never saves a snapshot when interrupted.
+
+https://github.com/restic/restic/issues/3151
+https://github.com/restic/restic/pull/3164

changelog/0.12.0_2021-02-14/issue-3166

@@ -0,0 +1,9 @@
+Bugfix: Improve error handling in the `restore` command
+
+The `restore` command used to not print errors while downloading file contents
+from the repository. It also incorrectly exited with a zero error code even
+when there were errors during the restore process. This has all been fixed and
+`restore` now returns with a non-zero exit code when there's an error.
+
+https://github.com/restic/restic/issues/3166
+https://github.com/restic/restic/pull/3207

changelog/0.12.0_2021-02-14/issue-3191

@@ -0,0 +1,8 @@
+Enhancement: Add release binaries for MIPS architectures
+
+We've added a few new architectures for Linux to the release binaries: `mips`,
+`mipsle`, `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded
+systems.
+
+https://github.com/restic/restic/issues/3191
+https://github.com/restic/restic/pull/3208

changelog/0.12.0_2021-02-14/issue-3232

@@ -0,0 +1,11 @@
+Bugfix: Correct statistics for overlapping targets
+
+A user reported that restic's statistics and progress information during backup
+was not correctly calculated when the backup targets (files/dirs to save)
+overlap. For example, consider a directory `foo` which contains (among others)
+a file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the
+size of the file `foo/bar` twice, so the completeness percentage as well as the
+number of files was wrong. This is now corrected.
+
+https://github.com/restic/restic/issues/3232
+https://github.com/restic/restic/pull/3243

changelog/0.12.0_2021-02-14/issue-909

@@ -0,0 +1,12 @@
+Enhancement: Back up mountpoints as empty directories
+
+When the `--one-file-system` option is specified to `restic backup`, it
+ignores all file systems mounted below one of the target directories. This
+means that when a snapshot is restored, users needed to manually recreate
+the mountpoint directories.
+
+Restic now backs up mountpoints as empty directories and therefore implements
+the same approach as `tar`.
+
+https://github.com/restic/restic/issues/909
+https://github.com/restic/restic/pull/3119

changelog/0.12.0_2021-02-14/pr-3250

@@ -0,0 +1,6 @@
+Enhancement: Add several more error checks
+
+We've added a lot more error checks in places where errors were previously
+ignored (as hinted by the static analysis program `errcheck` via `golangci-lint`).
+
+https://github.com/restic/restic/pull/3250

changelog/0.12.0_2021-02-14/pull-2718

@@ -0,0 +1,29 @@
+Enhancement: Improve `prune` performance and make it more customizable
+
+The `prune` command is now much faster. This is especially the case for remote
+repositories or repositories with not much data to remove. Also the memory
+usage of the `prune` command is now reduced.
+
+Restic used to rebuild the index from scratch after pruning. This could lead
+to missing packs in the index in some cases for eventually consistent backends
+such as e.g. AWS S3. This behavior is now changed and the index rebuilding
+uses the information already known by `prune`.
+
+By default, the `prune` command no longer removes all unused data. This
+behavior can be fine-tuned by new options, like the acceptable amount of
+unused space or the maximum size of data to reorganize. For more details,
+please see https://restic.readthedocs.io/en/stable/060_forget.html .
+
+Moreover, `prune` now accepts the `--dry-run` option and also running
+`forget --dry-run --prune` will show what `prune` would do.
+
+This enhancement also fixes several open issues, e.g.:
+- https://github.com/restic/restic/issues/1140
+- https://github.com/restic/restic/issues/1599
+- https://github.com/restic/restic/issues/1985
+- https://github.com/restic/restic/issues/2112
+- https://github.com/restic/restic/issues/2227
+- https://github.com/restic/restic/issues/2305
+
+https://github.com/restic/restic/pull/2718
+https://github.com/restic/restic/pull/2842

changelog/0.12.0_2021-02-14/pull-2823

@@ -0,0 +1,27 @@
+Enhancement: Add option to let `backup` trust mtime without checking ctime
+
+The `backup` command used to require that both `ctime` and `mtime` of a file
+matched with a previously backed up version to determine that the file was
+unchanged. In other words, if either `ctime` or `mtime` of the file had
+changed, it would be considered changed and restic would read the file's
+content again to back up the relevant (changed) parts of it.
+
+The new option `--ignore-ctime` makes restic look at `mtime` only, such that
+`ctime` changes for a file does not cause restic to read the file's contents
+again.
+
+The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make
+backups more reliable in the face of programs that reset `mtime` (some Unix
+archivers do that), but it turned out to often be expensive because it made
+restic read file contents even if only the metadata (owner, permissions) of
+a file had changed. The new `--ignore-ctime` option lets the user restore the
+0.9.5 behavior when needed. The existing `--ignore-inode` option already
+turned off this behavior, but also removed a different check.
+
+Please note that changes in files' metadata are still recorded, regardless of
+the command line options provided to the backup command.
+
+https://github.com/restic/restic/issues/2495
+https://github.com/restic/restic/issues/2558
+https://github.com/restic/restic/issues/2819
+https://github.com/restic/restic/pull/2823

changelog/0.12.0_2021-02-14/pull-2941

@@ -0,0 +1,8 @@
+Enhancement: Speed up the repacking step of the `prune` command
+
+The repack step of the `prune` command, which moves still used file parts into
+new pack files such that the old ones can be garbage collected later on, now
+processes multiple pack files in parallel. This is especially beneficial for
+high latency backends or when using a fast network connection.
+
+https://github.com/restic/restic/pull/2941

changelog/0.12.0_2021-02-14/pull-3006

@@ -0,0 +1,11 @@
+Enhancement: Speed up the `rebuild-index` command
+
+We've optimized the `rebuild-index` command. Now, existing index entries are used
+to minimize the number of pack files that must be read. This speeds up the index
+rebuild a lot.
+
+Additionally, the option `--read-all-packs` has been added, implementing the
+previous behavior.
+
+https://github.com/restic/restic/issue/2547
+https://github.com/restic/restic/pull/3006

changelog/0.12.0_2021-02-14/pull-3014

@@ -0,0 +1,13 @@
+Bugfix: Fix sporadic stream reset between rclone and restic
+
+Sometimes when using restic with the `rclone` backend, an error message
+similar to the following would be printed:
+
+    Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
+
+It was found that this was caused by restic closing the connection to rclone
+to soon when downloading data. A workaround has been added which waits for
+the end of the download before closing the connection.
+
+https://github.com/restic/restic/pull/3014
+https://github.com/rclone/rclone/issues/2598

changelog/0.12.0_2021-02-14/pull-3048

@@ -0,0 +1,23 @@
+Enhancement: Add more checks for index and pack files in the `check` command
+
+The `check` command run with the `--read-data` or `--read-data-subset` options
+used to only verify only the pack file content - it did not check if the blobs
+within the pack are correctly contained in the index.
+
+A check for the latter is now in place, which can print the following error:
+
+    Blob ID is not contained in index or position is incorrect
+
+Another test is also added, which compares pack file sizes computed from the
+index and the pack header with the actual file size. This test is able to
+detect truncated pack files.
+
+If the index is not correct, it can be rebuilt by using the `rebuild-index`
+command.
+
+Having added these tests, `restic check` is now able to detect non-existing
+blobs which are wrongly referenced in the index. This situation could have
+lead to missing data.
+
+https://github.com/restic/restic/pull/3048
+https://github.com/restic/restic/pull/3082

changelog/0.12.0_2021-02-14/pull-3081

@@ -0,0 +1,8 @@
+Enhancement: Make the `dump` command support `zip` format
+
+Previously, restic could dump the contents of a whole folder structure only
+in the `tar` format. The `dump` command now has a new flag to change output
+format to `zip`. Just pass `--archive zip` as an option to `restic dump`.
+
+https://github.com/restic/restic/pull/2433
+https://github.com/restic/restic/pull/3081

changelog/0.12.0_2021-02-14/pull-3099

@@ -0,0 +1,6 @@
+Enhancement: Reduce memory usage of `check` command
+
+The `check` command now requires less memory if it is run without the
+`--check-unused` option.
+
+https://github.com/restic/restic/pull/3099

changelog/0.12.0_2021-02-14/pull-3106

@@ -0,0 +1,10 @@
+Enhancement: Parallelize scan of snapshot content in `copy` and `prune`
+
+The `copy` and `prune` commands used to traverse the directories of
+snapshots one by one to find used data. This snapshot traversal is
+now parallized which can speed up this step several times.
+
+In addition the `check` command now reports how many snapshots have
+already been processed.
+
+https://github.com/restic/restic/pull/3106

changelog/0.12.0_2021-02-14/pull-3130

@@ -0,0 +1,13 @@
+Enhancement: Parallelize reading of locks and snapshots
+
+Restic used to read snapshots sequentially. For repositories containing
+many snapshots this slowed down commands which have to read all snapshots.
+
+Now the reading of snapshots is parallelized. This speeds up for example
+`prune`, `backup` and other commands that search for snapshots with certain
+properties or which have to find the `latest` snapshot.
+
+The speed up also applies to locks stored in the backup repository.
+
+https://github.com/restic/restic/pull/3130
+https://github.com/restic/restic/pull/3174

changelog/0.12.0_2021-02-14/pull-3152

@@ -0,0 +1,8 @@
+Bugfix: Do not hang until foregrounded when completed in background
+
+On Linux, when running in the background restic failed to stop the terminal
+output of the `backup` command after it had completed. This caused restic to
+hang until moved to the foreground. This has now been fixed.
+
+https://github.com/restic/restic/pull/3152
+https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334

changelog/0.12.0_2021-02-14/pull-3249

@@ -0,0 +1,7 @@
+Bugfix: Improve error handling in `gs` backend
+
+The `gs` backend did not notice when the last step of completing a
+file upload failed. Under rare circumstances, this could cause
+missing files in the backup repository. This has now been fixed.
+
+https://github.com/restic/restic/pull/3249

changelog/0.12.0_2021-02-14/pull-3254

@@ -0,0 +1,8 @@
+Enhancement: Enable HTTP/2 for backend connections
+
+Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2
+depending on what the server supports. But for compatibility this mechanism
+is disabled if DialContext is used (which is the case for restic). This change
+allows restic's HTTP client to negotiate HTTP/2 if supported by the server.
+
+https://github.com/restic/restic/pull/3254

changelog/TEMPLATE

@@ -1,11 +1,21 @@
-Bugfix: Fix behavior for foobar (in present tense)
+# The first line must start with Bugfix:, Enhancement: or Change:,
+# including the colon. Use present use. Remove lines starting with '#'
+# from this template.
+Enhancement: Allow custom bar in the foo command
 
-We've fixed the behavior for foobar, a long-standing annoyance for restic
-users.
+# Describe the problem in the past tense, the new behavior in the present
+# tense. Mention the affected commands, backends, operating systems, etc.
+# Focus on user-facing behavior, not the implementation.
 
-The text in the paragraphs is written in past tense. The last section is a list
-of issue URLs, PR URLs and other URLs. The first issue ID (or the first PR ID,
-in case there aren't any issue links) is used as the primary ID.
+Restic foo always used the system-wide bar when deciding how to frob an
+item in the baz backend. It now permits selecting the bar with --bar or
+the environment variable RESTIC_BAR. The system-wide bar is still the
+default.
+
+# The last section is a list of issue, PR and forum URLs.
+# The first issue ID determines the filename for the changelog entry:
+# changelog/unreleased/issue-1234. If there are no relevant issue links,
+# use the PR ID and call the file pull-55555.
 
 https://github.com/restic/restic/issues/1234
 https://github.com/restic/restic/pull/55555

changelog/unreleased/.gitignore

@@ -0,0 +1,2 @@
+# this file is here so the unreleased/ directory is tracked within git, even if
+# no other changelog files are present.

cmd/restic/cmd_backup.go

@@ -10,7 +10,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strconv"
+	"runtime"
 	"strings"
 	"time"
 
@@ -55,14 +55,6 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if backupOptions.Stdin {
-			for _, filename := range backupOptions.FilesFrom {
-				if filename == "-" {
-					return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
-				}
-			}
-		}
-
 		var t tomb.Tomb
 		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
 		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
@@ -90,12 +82,16 @@ type BackupOptions struct {
 	ExcludeLargerThan       string
 	Stdin                   bool
 	StdinFilename           string
-	Tags                    []string
+	Tags                    restic.TagLists
 	Host                    string
 	FilesFrom               []string
+	FilesFromVerbatim       []string
+	FilesFromRaw            []string
 	TimeStamp               string
 	WithAtime               bool
 	IgnoreInode             bool
+	IgnoreCtime             bool
+	UseFsSnapshot           bool
 }
 
 var backupOptions BackupOptions
@@ -119,16 +115,26 @@ func init() {
 	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
-	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
 
 	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
 	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
-	f.MarkDeprecated("hostname", "use --host")
+	err := f.MarkDeprecated("hostname", "use --host")
+	if err != nil {
+		// MarkDeprecated only returns an error when the flag could not be found
+		panic(err)
+	}
 
-	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args/can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
+	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	if runtime.GOOS == "windows" {
+		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
+	}
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -151,11 +157,13 @@ func filterExisting(items []string) (result []string, err error) {
 	return
 }
 
-// readFromFile will read all lines from the given filename and return them as
-// a string array, if filename is empty readFromFile returns and empty string
-// array. If filename is a dash (-), readFromFile will read the lines from the
+// readLines reads all lines from the named file and returns them as a
+// string slice.
+//
+// If filename is empty, readPatternsFromFile returns an empty slice.
+// If filename is a dash (-), readPatternsFromFile will read the lines from the
 // standard input.
-func readLinesFromFile(filename string) ([]string, error) {
+func readLines(filename string) ([]string, error) {
 	if filename == "" {
 		return nil, nil
 	}
@@ -179,29 +187,72 @@ func readLinesFromFile(filename string) ([]string, error) {
 
 	scanner := bufio.NewScanner(bytes.NewReader(data))
 	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		// ignore empty lines
-		if line == "" {
-			continue
-		}
-		// strip comments
-		if strings.HasPrefix(line, "#") {
-			continue
-		}
-		lines = append(lines, line)
+		lines = append(lines, scanner.Text())
 	}
 
 	if err := scanner.Err(); err != nil {
 		return nil, err
 	}
-
 	return lines, nil
 }
 
+// readFilenamesFromFileRaw reads a list of filenames from the given file,
+// or stdin if filename is "-". Each filename is terminated by a zero byte,
+// which is stripped off.
+func readFilenamesFromFileRaw(filename string) (names []string, err error) {
+	f := os.Stdin
+	if filename != "-" {
+		if f, err = os.Open(filename); err != nil {
+			return nil, err
+		}
+	}
+
+	names, err = readFilenamesRaw(f)
+	if err != nil {
+		// ignore subsequent errors
+		_ = f.Close()
+		return nil, err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return names, nil
+}
+
+func readFilenamesRaw(r io.Reader) (names []string, err error) {
+	br := bufio.NewReader(r)
+	for {
+		name, err := br.ReadString(0)
+		switch err {
+		case nil:
+		case io.EOF:
+			if name == "" {
+				return names, nil
+			}
+			return nil, errors.Fatal("--files-from-raw: trailing zero byte missing")
+		default:
+			return nil, err
+		}
+
+		name = name[:len(name)-1]
+		if name == "" {
+			// The empty filename is never valid. Handle this now to
+			// prevent downstream code from erroneously backing up
+			// filepath.Clean("") == ".".
+			return nil, errors.Fatal("--files-from-raw: empty filename in listing")
+		}
+		names = append(names, name)
+	}
+}
+
 // Check returns an error when an invalid combination of options was set.
 func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 	if gopts.password == "" {
-		for _, filename := range opts.FilesFrom {
+		filesFrom := append(append(opts.FilesFrom, opts.FilesFromVerbatim...), opts.FilesFromRaw...)
+		for _, filename := range filesFrom {
 			if filename == "-" {
 				return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 			}
@@ -212,6 +263,12 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 		if len(opts.FilesFrom) > 0 {
 			return errors.Fatal("--stdin and --files-from cannot be used together")
 		}
+		if len(opts.FilesFromVerbatim) > 0 {
+			return errors.Fatal("--stdin and --files-from-verbatim cannot be used together")
+		}
+		if len(opts.FilesFromRaw) > 0 {
+			return errors.Fatal("--stdin and --files-from-raw cannot be used together")
+		}
 
 		if len(args) > 0 {
 			return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
@@ -351,15 +408,19 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 		return nil, nil
 	}
 
-	var lines []string
 	for _, file := range opts.FilesFrom {
-		fromfile, err := readLinesFromFile(file)
+		fromfile, err := readLines(file)
 		if err != nil {
 			return nil, err
 		}
 
 		// expand wildcards
 		for _, line := range fromfile {
+			line = strings.TrimSpace(line)
+			if line == "" || line[0] == '#' { // '#' marks a comment.
+				continue
+			}
+
 			var expanded []string
 			expanded, err := filepath.Glob(line)
 			if err != nil {
@@ -368,19 +429,38 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 			if len(expanded) == 0 {
 				Warnf("pattern %q does not match any files, skipping\n", line)
 			}
-			lines = append(lines, expanded...)
+			targets = append(targets, expanded...)
 		}
 	}
 
-	// merge files from files-from into normal args so we can reuse the normal
-	// args checks and have the ability to use both files-from and args at the
-	// same time
-	args = append(args, lines...)
-	if len(args) == 0 && !opts.Stdin {
+	for _, file := range opts.FilesFromVerbatim {
+		fromfile, err := readLines(file)
+		if err != nil {
+			return nil, err
+		}
+		for _, line := range fromfile {
+			if line == "" {
+				continue
+			}
+			targets = append(targets, line)
+		}
+	}
+
+	for _, file := range opts.FilesFromRaw {
+		fromfile, err := readFilenamesFromFileRaw(file)
+		if err != nil {
+			return nil, err
+		}
+		targets = append(targets, fromfile...)
+	}
+
+	// Merge args into files-from so we can reuse the normal args checks
+	// and have the ability to use both files-from and args at the same time.
+	targets = append(targets, args...)
+	if len(targets) == 0 && !opts.Stdin {
 		return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
 	}
 
-	targets = args
 	targets, err = filterExisting(targets)
 	if err != nil {
 		return nil, err
@@ -394,7 +474,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
 	// Force using a parent
 	if !opts.Force && opts.Parent != "" {
-		id, err := restic.FindSnapshot(repo, opts.Parent)
+		id, err := restic.FindSnapshot(ctx, repo, opts.Parent)
 		if err != nil {
 			return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
 		}
@@ -481,22 +561,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}()
 	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
 
-	if s, ok := os.LookupEnv("RESTIC_PROGRESS_FPS"); ok {
-		fps, err := strconv.Atoi(s)
-		if err == nil && fps >= 1 {
-			if fps > 60 {
-				fps = 60
-			}
-			p.SetMinUpdatePause(time.Second / time.Duration(fps))
-		}
-	}
+	p.SetMinUpdatePause(calculateProgressInterval())
 
 	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
 
 	if !gopts.JSON {
 		p.V("lock repository")
 	}
-	lock, err := lockRepo(repo)
+	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -527,8 +599,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	if !gopts.JSON && parentSnapshotID != nil {
-		p.V("using parent snapshot %v\n", parentSnapshotID.Str())
+	if !gopts.JSON {
+		if parentSnapshotID != nil {
+			p.P("using parent snapshot %v\n", parentSnapshotID.Str())
+		} else {
+			p.P("no parent snapshot found, will read all files\n")
+		}
 	}
 
 	selectByNameFilter := func(item string) bool {
@@ -550,6 +626,25 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	}
 
 	var targetFS fs.FS = fs.Local{}
+	if runtime.GOOS == "windows" && opts.UseFsSnapshot {
+		if err = fs.HasSufficientPrivilegesForVSS(); err != nil {
+			return err
+		}
+
+		errorHandler := func(item string, err error) error {
+			return p.Error(item, nil, err)
+		}
+
+		messageHandler := func(msg string, args ...interface{}) {
+			if !gopts.JSON {
+				p.P(msg, args...)
+			}
+		}
+
+		localVss := fs.NewLocalVss(errorHandler, messageHandler)
+		defer localVss.DeleteSnapshots()
+		targetFS = localVss
+	}
 	if opts.Stdin {
 		if !gopts.JSON {
 			p.V("read data from stdin")
@@ -587,7 +682,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 	arch.CompleteItem = p.CompleteItem
 	arch.StartFile = p.StartFile
 	arch.CompleteBlob = p.CompleteBlob
-	arch.IgnoreInode = opts.IgnoreInode
+
+	if opts.IgnoreInode {
+		// --ignore-inode implies --ignore-ctime: on FUSE, the ctime is not
+		// reliable either.
+		arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime | archiver.ChangeIgnoreInode
+	}
+	if opts.IgnoreCtime {
+		arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
+	}
 
 	if parentSnapshotID == nil {
 		parentSnapshotID = &restic.ID{}
@@ -595,7 +698,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 
 	snapshotOpts := archiver.SnapshotOptions{
 		Excludes:       opts.Excludes,
-		Tags:           opts.Tags,
+		Tags:           opts.Tags.Flatten(),
 		Time:           timeStamp,
 		Hostname:       opts.Host,
 		ParentSnapshot: *parentSnapshotID,

cmd/restic/cmd_backup_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestCollectTargets(t *testing.T) {
+	dir, cleanup := rtest.TempDir(t)
+	defer cleanup()
+
+	fooSpace := "foo "
+	barStar := "bar*"              // Must sort before the others, below.
+	if runtime.GOOS == "windows" { // Doesn't allow "*" or trailing space.
+		fooSpace = "foo"
+		barStar = "bar"
+	}
+
+	var expect []string
+	for _, filename := range []string{
+		barStar, "baz", "cmdline arg", fooSpace,
+		"fromfile", "fromfile-raw", "fromfile-verbatim", "quux",
+	} {
+		// All mentioned files must exist for collectTargets.
+		f, err := os.Create(filepath.Join(dir, filename))
+		rtest.OK(t, err)
+		rtest.OK(t, f.Close())
+
+		expect = append(expect, f.Name())
+	}
+
+	f1, err := os.Create(filepath.Join(dir, "fromfile"))
+	rtest.OK(t, err)
+	// Empty lines should be ignored. A line starting with '#' is a comment.
+	fmt.Fprintf(f1, "\n%s*\n # here's a comment\n", f1.Name())
+	rtest.OK(t, f1.Close())
+
+	f2, err := os.Create(filepath.Join(dir, "fromfile-verbatim"))
+	rtest.OK(t, err)
+	for _, filename := range []string{fooSpace, barStar} {
+		// Empty lines should be ignored. CR+LF is allowed.
+		fmt.Fprintf(f2, "%s\r\n\n", filepath.Join(dir, filename))
+	}
+	rtest.OK(t, f2.Close())
+
+	f3, err := os.Create(filepath.Join(dir, "fromfile-raw"))
+	rtest.OK(t, err)
+	for _, filename := range []string{"baz", "quux"} {
+		fmt.Fprintf(f3, "%s\x00", filepath.Join(dir, filename))
+	}
+	rtest.OK(t, err)
+	rtest.OK(t, f3.Close())
+
+	opts := BackupOptions{
+		FilesFrom:         []string{f1.Name()},
+		FilesFromVerbatim: []string{f2.Name()},
+		FilesFromRaw:      []string{f3.Name()},
+	}
+
+	targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")})
+	rtest.OK(t, err)
+	sort.Strings(targets)
+	rtest.Equals(t, expect, targets)
+}
+
+func TestReadFilenamesRaw(t *testing.T) {
+	// These should all be returned exactly as-is.
+	expected := []string{
+		"\xef\xbb\xbf/utf-8-bom",
+		"/absolute",
+		"../.././relative",
+		"\t\t leading and trailing space   \t\t",
+		"newline\nin filename",
+		"not UTF-8: \x80\xff/simple",
+		` / *[]* \ `,
+	}
+
+	var buf bytes.Buffer
+	for _, name := range expected {
+		buf.WriteString(name)
+		buf.WriteByte(0)
+	}
+
+	got, err := readFilenamesRaw(&buf)
+	rtest.OK(t, err)
+	rtest.Equals(t, expected, got)
+
+	// Empty input is ok.
+	got, err = readFilenamesRaw(strings.NewReader(""))
+	rtest.OK(t, err)
+	rtest.Equals(t, 0, len(got))
+
+	// An empty filename is an error.
+	_, err = readFilenamesRaw(strings.NewReader("foo\x00\x00"))
+	rtest.Assert(t, err != nil, "no error for zero byte")
+	rtest.Assert(t, strings.Contains(err.Error(), "empty filename"),
+		"wrong error message: %v", err.Error())
+
+	// No trailing NUL byte is an error, because it likely means we're
+	// reading a line-oriented text file (someone forgot -print0).
+	_, err = readFilenamesRaw(strings.NewReader("simple.txt"))
+	rtest.Assert(t, err != nil, "no error for zero byte")
+	rtest.Assert(t, strings.Contains(err.Error(), "zero byte"),
+		"wrong error message: %v", err.Error())
+}

cmd/restic/cmd_cache.go

@@ -51,7 +51,7 @@ func init() {
 
 func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 	if len(args) > 0 {
-		return errors.Fatal("the cache command has no arguments")
+		return errors.Fatal("the cache command expects no arguments, only options - please see `restic help cache` for usage and flags")
 	}
 
 	if gopts.NoCache {
@@ -148,7 +148,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		})
 	}
 
-	tab.Write(gopts.stdout)
+	_ = tab.Write(gopts.stdout)
 	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
 
 	return nil

cmd/restic/cmd_cat.go

@@ -42,10 +42,13 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	lock, err := lockRepo(repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
+	if !gopts.NoLock {
+		lock, err := lockRepo(gopts.ctx, repo)
+		if err != nil {
+			return err
+		}
+
+		defer unlockRepo(lock)
 	}
 
 	tpe := args[0]
@@ -59,7 +62,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			}
 
 			// find snapshot id with prefix
-			id, err = restic.FindSnapshot(repo, args[1])
+			id, err = restic.FindSnapshot(gopts.ctx, repo, args[1])
 			if err != nil {
 				return errors.Fatalf("could not find snapshot: %v\n", err)
 			}
@@ -165,7 +168,8 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 	case "blob":
 		for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
-			if !repo.Index().Has(id, t) {
+			bh := restic.BlobHandle{ID: id, Type: t}
+			if !repo.Index().Has(bh) {
 				continue
 			}
 

cmd/restic/cmd_check.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -53,25 +54,38 @@ func init() {
 
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
-	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read subset n of m data packs (format: `n/m`)")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific subset or either 'x%' or 'x.y%' for random subset")
 	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
 
 func checkFlags(opts CheckOptions) error {
 	if opts.ReadData && opts.ReadDataSubset != "" {
-		return errors.Fatalf("check flags --read-data and --read-data-subset cannot be used together")
+		return errors.Fatal("check flags --read-data and --read-data-subset cannot be used together")
 	}
 	if opts.ReadDataSubset != "" {
 		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
-		if err != nil || len(dataSubset) != 2 {
-			return errors.Fatalf("check flag --read-data-subset must have two positive integer values, e.g. --read-data-subset=1/2")
-		}
-		if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
-			return errors.Fatalf("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
-		}
-		if dataSubset[1] > totalBucketsMax {
-			return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
+		argumentError := errors.Fatal("check flag --read-data-subset must have two positive integer values or a percentage, e.g. --read-data-subset=1/2 or --read-data-subset=2.5%%")
+		if err == nil {
+			if len(dataSubset) != 2 {
+				return argumentError
+			}
+			if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
+				return errors.Fatal("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
+			}
+			if dataSubset[1] > totalBucketsMax {
+				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
+			}
+		} else {
+			percentage, err := parsePercentage(opts.ReadDataSubset)
+			if err != nil {
+				return argumentError
+			}
+
+			if percentage <= 0.0 || percentage > 100.0 {
+				return errors.Fatal(
+					"check flag --read-data-subset=n% n must be above 0.0% and at most 100.0%")
+			}
 		}
 	}
 
@@ -98,6 +112,21 @@ func stringToIntSlice(param string) (split []uint, err error) {
 	return result, nil
 }
 
+// ParsePercentage parses a percentage string of the form "X%" where X is a float constant,
+// and returns the value of that constant. It does not check the range of the value.
+func parsePercentage(s string) (float64, error) {
+	if !strings.HasSuffix(s, "%") {
+		return 0, errors.Errorf(`parsePercentage: %q does not end in "%%"`, s)
+	}
+	s = s[:len(s)-1]
+
+	p, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return 0, errors.Errorf("parsePercentage: %v", err)
+	}
+	return p, nil
+}
+
 // prepareCheckCache configures a special cache directory for check.
 //
 //  * if --with-cache is specified, the default cache is used
@@ -142,7 +171,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
 
 func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
-		return errors.Fatal("check has no arguments")
+		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
 	}
 
 	cleanup := prepareCheckCache(opts, &gopts)
@@ -158,14 +187,14 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		lock, err := lockRepoExclusive(repo)
+		lock, err := lockRepoExclusive(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	chkr := checker.New(repo)
+	chkr := checker.New(repo, opts.CheckUnused)
 
 	Verbosef("load indexes\n")
 	hints, errs := chkr.LoadIndex(gopts.ctx)
@@ -212,7 +241,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
-	go chkr.Structure(gopts.ctx, errChan)
+	go func() {
+		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
+		defer bar.Done()
+		chkr.Structure(gopts.ctx, bar, errChan)
+	}()
 
 	for err := range errChan {
 		errorsFound = true
@@ -227,29 +260,15 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if opts.CheckUnused {
-		for _, id := range chkr.UnusedBlobs() {
+		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
 			Verbosef("unused blob %v\n", id)
 			errorsFound = true
 		}
 	}
 
-	doReadData := func(bucket, totalBuckets uint) {
-		packs := restic.IDSet{}
-		for pack := range chkr.GetPacks() {
-			// If we ever check more than the first byte
-			// of pack, update totalBucketsMax.
-			if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
-				packs.Insert(pack)
-			}
-		}
+	doReadData := func(packs map[restic.ID]int64) {
 		packCount := uint64(len(packs))
 
-		if packCount < chkr.CountPacks() {
-			Verbosef(fmt.Sprintf("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets))
-		} else {
-			Verbosef("read all data\n")
-		}
-
 		p := newProgressMax(!gopts.Quiet, packCount, "packs")
 		errChan := make(chan error)
 
@@ -259,14 +278,31 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 			errorsFound = true
 			Warnf("%v\n", err)
 		}
+		p.Done()
 	}
 
 	switch {
 	case opts.ReadData:
-		doReadData(1, 1)
+		Verbosef("read all data\n")
+		doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
 	case opts.ReadDataSubset != "":
-		dataSubset, _ := stringToIntSlice(opts.ReadDataSubset)
-		doReadData(dataSubset[0], dataSubset[1])
+		var packs map[restic.ID]int64
+		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
+		if err == nil {
+			bucket := dataSubset[0]
+			totalBuckets := dataSubset[1]
+			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
+			packCount := uint64(len(packs))
+			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
+		} else {
+			percentage, _ := parsePercentage(opts.ReadDataSubset)
+			packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
+			Verbosef("read %.1f%% of data packs\n", percentage)
+		}
+		if packs == nil {
+			return errors.Fatal("internal error: failed to select packs to check")
+		}
+		doReadData(packs)
 	}
 
 	if errorsFound {
@@ -277,3 +313,42 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	return nil
 }
+
+// selectPacksByBucket selects subsets of packs by ranges of buckets.
+func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 {
+	packs := make(map[restic.ID]int64)
+	for pack, size := range allPacks {
+		// If we ever check more than the first byte
+		// of pack, update totalBucketsMax.
+		if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
+			packs[pack] = size
+		}
+	}
+	return packs
+}
+
+// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
+func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
+	packCount := len(allPacks)
+	packsToCheck := int(float64(packCount) * (percentage / 100.0))
+	if packsToCheck < 1 {
+		packsToCheck = 1
+	}
+	timeNs := time.Now().UnixNano()
+	r := rand.New(rand.NewSource(timeNs))
+	idx := r.Perm(packCount)
+
+	var keys []restic.ID
+	for k := range allPacks {
+		keys = append(keys, k)
+	}
+
+	packs := make(map[restic.ID]int64)
+
+	for i := 0; i < packsToCheck; i++ {
+		id := keys[idx[i]]
+		packs[id] = allPacks[id]
+	}
+
+	return packs
+}

cmd/restic/cmd_check_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"math"
+	"reflect"
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestParsePercentage(t *testing.T) {
+	testCases := []struct {
+		input       string
+		output      float64
+		expectError bool
+	}{
+		{"0%", 0.0, false},
+		{"1%", 1.0, false},
+		{"100%", 100.0, false},
+		{"123%", 123.0, false},
+		{"123.456%", 123.456, false},
+		{"0.742%", 0.742, false},
+		{"-100%", -100.0, false},
+		{" 1%", 0.0, true},
+		{"1 %", 0.0, true},
+		{"1% ", 0.0, true},
+	}
+	for _, testCase := range testCases {
+		output, err := parsePercentage(testCase.input)
+
+		if testCase.expectError {
+			rtest.Assert(t, err != nil, "Expected error for case %s", testCase.input)
+			rtest.Assert(t, output == 0.0, "Expected output to be 0.0, got %s", output)
+		} else {
+			rtest.Assert(t, err == nil, "Expected no error for case %s", testCase.input)
+			rtest.Assert(t, math.Abs(testCase.output-output) < 0.00001, "Expected %f, got %f",
+				testCase.output, output)
+		}
+	}
+}
+
+func TestStringToIntSlice(t *testing.T) {
+	testCases := []struct {
+		input       string
+		output      []uint
+		expectError bool
+	}{
+		{"3/5", []uint{3, 5}, false},
+		{"1/100", []uint{1, 100}, false},
+		{"abc", nil, true},
+		{"1/a", nil, true},
+		{"/", nil, true},
+	}
+	for _, testCase := range testCases {
+		output, err := stringToIntSlice(testCase.input)
+
+		if testCase.expectError {
+			rtest.Assert(t, err != nil, "Expected error for case %s", testCase.input)
+			rtest.Assert(t, output == nil, "Expected output to be nil, got %s", output)
+		} else {
+			rtest.Assert(t, err == nil, "Expected no error for case %s", testCase.input)
+			rtest.Assert(t, len(output) == 2, "Invalid output length for case %s", testCase.input)
+			rtest.Assert(t, reflect.DeepEqual(output, testCase.output), "Expected %f, got %f",
+				testCase.output, output)
+		}
+	}
+}
+
+func TestSelectPacksByBucket(t *testing.T) {
+	var testPacks = make(map[restic.ID]int64)
+	for i := 1; i <= 10; i++ {
+		id := restic.NewRandomID()
+		// ensure relevant part of generated id is reproducable
+		id[0] = byte(i)
+		testPacks[id] = 0
+	}
+
+	selectedPacks := selectPacksByBucket(testPacks, 0, 10)
+	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
+
+	for i := uint(1); i <= 5; i++ {
+		selectedPacks = selectPacksByBucket(testPacks, i, 5)
+		rtest.Assert(t, len(selectedPacks) == 2, "Expected 2 selected packs")
+	}
+
+	selectedPacks = selectPacksByBucket(testPacks, 1, 1)
+	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
+	for testPack := range testPacks {
+		_, ok := selectedPacks[testPack]
+		rtest.Assert(t, ok, "Expected input and output to be equal")
+	}
+}
+
+func TestSelectRandomPacksByPercentage(t *testing.T) {
+	var testPacks = make(map[restic.ID]int64)
+	for i := 1; i <= 10; i++ {
+		testPacks[restic.NewRandomID()] = 0
+	}
+
+	selectedPacks := selectRandomPacksByPercentage(testPacks, 0.0)
+	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected packs")
+
+	selectedPacks = selectRandomPacksByPercentage(testPacks, 10.0)
+	rtest.Assert(t, len(selectedPacks) == 1, "Expected 1 selected pack")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected selection")
+	}
+
+	selectedPacks = selectRandomPacksByPercentage(testPacks, 50.0)
+	rtest.Assert(t, len(selectedPacks) == 5, "Expected 5 selected packs")
+	for pack := range selectedPacks {
+		_, ok := testPacks[pack]
+		rtest.Assert(t, ok, "Unexpected item in selection")
+	}
+
+	selectedPacks = selectRandomPacksByPercentage(testPacks, 100.0)
+	rtest.Assert(t, len(selectedPacks) == 10, "Expected 10 selected packs")
+	for testPack := range testPacks {
+		_, ok := selectedPacks[testPack]
+		rtest.Assert(t, ok, "Expected input and output to be equal")
+	}
+}

cmd/restic/cmd_copy.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/restic"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/spf13/cobra"
 )
@@ -14,12 +15,19 @@ var cmdCopy = &cobra.Command{
 	Use:   "copy [flags] [snapshotID ...]",
 	Short: "Copy snapshots from one repository to another",
 	Long: `
-The "copy" command copies one or more snapshots from one repository to another
-repository. Note that this will have to read (download) and write (upload) the
-entire snapshot(s) due to the different encryption keys on the source and
-destination, and that transferred files are not re-chunked, which may break
-their deduplication. This can be mitigated by the "--copy-chunker-params"
-option when initializing a new destination repository using the "init" command.
+The "copy" command copies one or more snapshots from one repository to another.
+
+NOTE: This process will have to both download (read) and upload (write) the
+entire snapshot(s) due to the different encryption keys used in the source and
+destination repositories. This /may incur higher bandwidth usage and costs/ than
+expected during normal backup runs.
+
+NOTE: The copying process does not re-chunk files, which may break deduplication
+between the files copied and files already stored in the destination repository.
+This means that copied files, which existed in both the source and destination
+repository, /may occupy up to twice their space/ in the destination repository.
+This can be mitigated by the "--copy-chunker-params" option when initializing a
+new destination repository using the "init" command.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCopy(copyOptions, globalOptions, args)
@@ -65,13 +73,13 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	srcLock, err := lockRepo(srcRepo)
+	srcLock, err := lockRepo(ctx, srcRepo)
 	defer unlockRepo(srcLock)
 	if err != nil {
 		return err
 	}
 
-	dstLock, err := lockRepo(dstRepo)
+	dstLock, err := lockRepo(ctx, dstRepo)
 	defer unlockRepo(dstLock)
 	if err != nil {
 		return err
@@ -96,12 +104,8 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
 	}
 
-	cloner := &treeCloner{
-		srcRepo:      srcRepo,
-		dstRepo:      dstRepo,
-		visitedTrees: restic.NewIDSet(),
-		buf:          nil,
-	}
+	// remember already processed trees across all snapshots
+	visitedTrees := restic.NewIDSet()
 
 	for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
@@ -126,7 +130,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
 		}
 		Verbosef("  copy started, this may take a while...\n")
 
-		if err := cloner.copyTree(ctx, *sn.Tree); err != nil {
+		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree); err != nil {
 			return err
 		}
 		debug.Log("tree copied")
@@ -170,64 +174,64 @@ func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
 	return true
 }
 
-type treeCloner struct {
-	srcRepo      restic.Repository
-	dstRepo      restic.Repository
-	visitedTrees restic.IDSet
-	buf          []byte
-}
+func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
+	visitedTrees restic.IDSet, rootTreeID restic.ID) error {
 
-func (t *treeCloner) copyTree(ctx context.Context, treeID restic.ID) error {
-	// We have already processed this tree
-	if t.visitedTrees.Has(treeID) {
+	wg, ctx := errgroup.WithContext(ctx)
+
+	treeStream := restic.StreamTrees(ctx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
+		visited := visitedTrees.Has(treeID)
+		visitedTrees.Insert(treeID)
+		return visited
+	}, nil)
+
+	wg.Go(func() error {
+		// reused buffer
+		var buf []byte
+
+		for tree := range treeStream {
+			if tree.Error != nil {
+				return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
+			}
+
+			// Do we already have this tree blob?
+			if !dstRepo.Index().Has(restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}) {
+				newTreeID, err := dstRepo.SaveTree(ctx, tree.Tree)
+				if err != nil {
+					return fmt.Errorf("SaveTree(%v) returned error %v", tree.ID.Str(), err)
+				}
+				// Assurance only.
+				if newTreeID != tree.ID {
+					return fmt.Errorf("SaveTree(%v) returned unexpected id %s", tree.ID.Str(), newTreeID.Str())
+				}
+			}
+
+			// TODO: parallelize blob down/upload
+
+			for _, entry := range tree.Nodes {
+				// Recursion into directories is handled by StreamTrees
+				// Copy the blobs for this file.
+				for _, blobID := range entry.Content {
+					// Do we already have this data blob?
+					if dstRepo.Index().Has(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) {
+						continue
+					}
+					debug.Log("Copying blob %s\n", blobID.Str())
+					var err error
+					buf, err = srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, buf)
+					if err != nil {
+						return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
+					}
+
+					_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
+					if err != nil {
+						return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
+					}
+				}
+			}
+
+		}
 		return nil
-	}
-
-	tree, err := t.srcRepo.LoadTree(ctx, treeID)
-	if err != nil {
-		return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
-	}
-	t.visitedTrees.Insert(treeID)
-
-	// Do we already have this tree blob?
-	if !t.dstRepo.Index().Has(treeID, restic.TreeBlob) {
-		newTreeID, err := t.dstRepo.SaveTree(ctx, tree)
-		if err != nil {
-			return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err)
-		}
-		// Assurance only.
-		if newTreeID != treeID {
-			return fmt.Errorf("SaveTree(%v) returned unexpected id %s", treeID.Str(), newTreeID.Str())
-		}
-	}
-
-	// TODO: parellize this stuff, likely only needed inside a tree.
-
-	for _, entry := range tree.Nodes {
-		// If it is a directory, recurse
-		if entry.Type == "dir" && entry.Subtree != nil {
-			if err := t.copyTree(ctx, *entry.Subtree); err != nil {
-				return err
-			}
-		}
-		// Copy the blobs for this file.
-		for _, blobID := range entry.Content {
-			// Do we already have this data blob?
-			if t.dstRepo.Index().Has(blobID, restic.DataBlob) {
-				continue
-			}
-			debug.Log("Copying blob %s\n", blobID.Str())
-			t.buf, err = t.srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, t.buf)
-			if err != nil {
-				return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
-			}
-
-			_, _, err = t.dstRepo.SaveBlob(ctx, restic.DataBlob, t.buf, blobID, false)
-			if err != nil {
-				return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
-			}
-		}
-	}
-
-	return nil
+	})
+	return wg.Wait()
 }

cmd/restic/cmd_debug.go

@@ -54,9 +54,8 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 	return err
 }
 
-func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error {
-	return repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error {
-		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
+func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
+	return restic.ForAllSnapshots(ctx, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
 		if err != nil {
 			return err
 		}
@@ -82,12 +81,12 @@ type Blob struct {
 	Offset uint            `json:"offset"`
 }
 
-func printPacks(repo *repository.Repository, wr io.Writer) error {
+func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
 
-	return repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
+	return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
 
-		blobs, err := pack.List(repo.Key(), restic.ReaderAt(repo.Backend(), h), size)
+		blobs, _, err := pack.List(repo.Key(), restic.ReaderAt(ctx, repo.Backend(), h), size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
 			return nil
@@ -110,11 +109,9 @@ func printPacks(repo *repository.Repository, wr io.Writer) error {
 	})
 }
 
-func dumpIndexes(repo restic.Repository, wr io.Writer) error {
-	return repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error {
+func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
+	return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
 		Printf("index_id: %v\n", id)
-
-		idx, err := repository.LoadIndex(context.TODO(), repo, id)
 		if err != nil {
 			return err
 		}
@@ -134,7 +131,7 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -145,20 +142,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
 
 	switch tpe {
 	case "indexes":
-		return dumpIndexes(repo, gopts.stdout)
+		return dumpIndexes(gopts.ctx, repo, gopts.stdout)
 	case "snapshots":
-		return debugPrintSnapshots(repo, gopts.stdout)
+		return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
 	case "packs":
-		return printPacks(repo, gopts.stdout)
+		return printPacks(gopts.ctx, repo, gopts.stdout)
 	case "all":
 		Printf("snapshots:\n")
-		err := debugPrintSnapshots(repo, gopts.stdout)
+		err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
 		if err != nil {
 			return err
 		}
 
 		Printf("\nindexes:\n")
-		err = dumpIndexes(repo, gopts.stdout)
+		err = dumpIndexes(gopts.ctx, repo, gopts.stdout)
 		if err != nil {
 			return err
 		}

cmd/restic/cmd_diff.go

@@ -53,11 +53,10 @@ func init() {
 }
 
 func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
-	id, err := restic.FindSnapshot(repo, desc)
+	id, err := restic.FindSnapshot(ctx, repo, desc)
 	if err != nil {
-		return nil, err
+		return nil, errors.Fatal(err.Error())
 	}
-
 	return restic.LoadSnapshot(ctx, repo, id)
 }
 
@@ -332,7 +331,7 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -365,6 +364,8 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	stats := NewDiffStats()
+	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
+	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
 
 	err = c.diffTree(ctx, stats, "/", *sn1.Tree, *sn2.Tree)
 	if err != nil {

cmd/restic/cmd_dump.go

@@ -21,8 +21,8 @@ var cmdDump = &cobra.Command{
 	Long: `
 The "dump" command extracts files from a snapshot from the repository. If a
 single file is selected, it prints its contents to stdout. Folders are output
-as a tar file containing the contents of the specified folder.  Pass "/" as
-file name to dump the whole snapshot as a tar file.
+as a tar (default) or zip file containing the contents of the specified folder.
+Pass "/" as file name to dump the whole snapshot as an archive file.
 
 The special snapshot "latest" can be used to use the latest snapshot in the
 repository.
@@ -40,9 +40,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // DumpOptions collects all options for the dump command.
 type DumpOptions struct {
-	Hosts []string
-	Paths []string
-	Tags  restic.TagLists
+	Hosts   []string
+	Paths   []string
+	Tags    restic.TagLists
+	Archive string
 }
 
 var dumpOptions DumpOptions
@@ -54,6 +55,7 @@ func init() {
 	flags.StringArrayVarP(&dumpOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
 	flags.Var(&dumpOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
 	flags.StringArrayVar(&dumpOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
 }
 
 func splitPath(p string) []string {
@@ -65,8 +67,7 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string) error {
-
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error {
 	if tree == nil {
 		return fmt.Errorf("called with a nil tree")
 	}
@@ -81,10 +82,10 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
-		if err := checkStdoutTar(); err != nil {
+		if err := checkStdoutArchive(); err != nil {
 			return err
 		}
-		return dump.WriteTar(ctx, repo, tree, "/", os.Stdout)
+		return writeDump(ctx, repo, tree, "/", os.Stdout)
 	}
 
 	item := filepath.Join(prefix, pathComponents[0])
@@ -100,16 +101,16 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:])
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump)
 			case dump.IsDir(node):
-				if err := checkStdoutTar(); err != nil {
+				if err := checkStdoutArchive(); err != nil {
 					return err
 				}
 				subtree, err := repo.LoadTree(ctx, *node.Subtree)
 				if err != nil {
 					return err
 				}
-				return dump.WriteTar(ctx, repo, subtree, item, os.Stdout)
+				return writeDump(ctx, repo, subtree, item, os.Stdout)
 			case l > 1:
 				return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
 			case !dump.IsFile(node):
@@ -127,6 +128,16 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("no file and no snapshot ID specified")
 	}
 
+	var wd dump.WriteDump
+	switch opts.Archive {
+	case "tar":
+		wd = dump.WriteTar
+	case "zip":
+		wd = dump.WriteZip
+	default:
+		return fmt.Errorf("unknown archive format %q", opts.Archive)
+	}
+
 	snapshotIDString := args[0]
 	pathToPrint := args[1]
 
@@ -140,7 +151,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -160,7 +171,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
-		id, err = restic.FindSnapshot(repo, snapshotIDString)
+		id, err = restic.FindSnapshot(ctx, repo, snapshotIDString)
 		if err != nil {
 			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
 		}
@@ -176,7 +187,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 		Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	err = printFromTree(ctx, tree, repo, "/", splittedPath)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, wd)
 	if err != nil {
 		Exitf(2, "cannot dump file: %v", err)
 	}
@@ -184,7 +195,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	return nil
 }
 
-func checkStdoutTar() error {
+func checkStdoutArchive() error {
 	if stdoutIsTerminal() {
 		return fmt.Errorf("stdout is the terminal, please redirect output")
 	}

cmd/restic/cmd_find.go

@@ -394,7 +394,6 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 					delete(f.blobIDs, idStr[:shortStr])
 				}
 				f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
-				break
 			}
 		}
 
@@ -465,7 +464,7 @@ func (f *Finder) findObjectPack(ctx context.Context, id string, t restic.BlobTyp
 		return
 	}
 
-	blobs := idx.Lookup(rid, t)
+	blobs := idx.Lookup(restic.BlobHandle{ID: rid, Type: t})
 	if len(blobs) == 0 {
 		Printf("Object %s not found in the index\n", rid.Str())
 		return
@@ -529,7 +528,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -564,7 +563,10 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if opts.PackID {
-		f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
+		err := f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
+		if err != nil {
+			return err
+		}
 	}
 
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {

cmd/restic/cmd_forget.go

@@ -68,27 +68,37 @@ func init() {
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
 	f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
 	f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
-	f.MarkDeprecated("hostname", "use --host")
+	err := f.MarkDeprecated("hostname", "use --host")
+	if err != nil {
+		// MarkDeprecated only returns an error when the flag is not found
+		panic(err)
+	}
 
 	f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
 
 	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
-	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact format")
+	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
 
 	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "string for grouping snapshots by host,paths,tags")
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
 	f.SortFlags = false
+	addPruneOptions(cmdForget)
 }
 
 func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
+	err := verifyPruneOptions(&pruneOptions)
+	if err != nil {
+		return err
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(repo)
+	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -204,8 +214,12 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	if len(removeSnIDs) > 0 && opts.Prune && !opts.DryRun {
-		return pruneRepository(gopts, repo)
+	if len(removeSnIDs) > 0 && opts.Prune {
+		if !gopts.JSON {
+			Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+		}
+		pruneOptions.DryRun = opts.DryRun
+		return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs)
 	}
 
 	return nil

cmd/restic/cmd_init.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"github.com/restic/chunker"
+	"github.com/restic/restic/internal/backend/location"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 
@@ -42,18 +43,19 @@ func init() {
 }
 
 func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
-	if gopts.Repo == "" {
-		return errors.Fatal("Please specify repository location (-r)")
-	}
-
 	chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
 	if err != nil {
 		return err
 	}
 
-	be, err := create(gopts.Repo, gopts.extended)
+	repo, err := ReadRepo(gopts)
 	if err != nil {
-		return errors.Fatalf("create repository at %s failed: %v\n", gopts.Repo, err)
+		return err
+	}
+
+	be, err := create(repo, gopts.extended)
+	if err != nil {
+		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 
 	gopts.password, err = ReadPasswordTwice(gopts,
@@ -67,10 +69,10 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 
 	err = s.Init(gopts.ctx, gopts.password, chunkerPolynomial)
 	if err != nil {
-		return errors.Fatalf("create key in repository at %s failed: %v\n", gopts.Repo, err)
+		return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 
-	Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], gopts.Repo)
+	Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
 	Verbosef("\n")
 	Verbosef("Please note that knowledge of your password is required to access\n")
 	Verbosef("the repository. Losing your password means that your data is\n")

cmd/restic/cmd_key.go

@@ -188,7 +188,7 @@ func runKey(gopts GlobalOptions, args []string) error {
 
 	switch args[0] {
 	case "list":
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -196,7 +196,7 @@ func runKey(gopts GlobalOptions, args []string) error {
 
 		return listKeys(ctx, repo, gopts)
 	case "add":
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -204,20 +204,20 @@ func runKey(gopts GlobalOptions, args []string) error {
 
 		return addKey(gopts, repo)
 	case "remove":
-		lock, err := lockRepoExclusive(repo)
+		lock, err := lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 
-		id, err := restic.Find(repo.Backend(), restic.KeyFile, args[1])
+		id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1])
 		if err != nil {
 			return err
 		}
 
 		return deleteKey(gopts.ctx, repo, id)
 	case "passwd":
-		lock, err := lockRepoExclusive(repo)
+		lock, err := lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err

cmd/restic/cmd_list.go

@@ -40,7 +40,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 	}
 
 	if !opts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(opts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -60,8 +60,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		return repo.List(opts.ctx, restic.IndexFile, func(id restic.ID, size int64) error {
-			idx, err := repository.LoadIndex(opts.ctx, repo, id)
+		return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
 			if err != nil {
 				return err
 			}
@@ -70,7 +69,6 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 			}
 			return nil
 		})
-
 	default:
 		return errors.Fatal("invalid type")
 	}

cmd/restic/cmd_ls.go

@@ -159,16 +159,19 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		enc := json.NewEncoder(gopts.stdout)
 
 		printSnapshot = func(sn *restic.Snapshot) {
-			enc.Encode(lsSnapshot{
+			err = enc.Encode(lsSnapshot{
 				Snapshot:   sn,
 				ID:         sn.ID(),
 				ShortID:    sn.ID().Str(),
 				StructType: "snapshot",
 			})
+			if err != nil {
+				Warnf("JSON encode failed: %v\n", err)
+			}
 		}
 
 		printNode = func(path string, node *restic.Node) {
-			enc.Encode(lsNode{
+			err = enc.Encode(lsNode{
 				Name:       node.Name,
 				Type:       node.Type,
 				Path:       path,
@@ -181,6 +184,9 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 				ChangeTime: node.ChangeTime,
 				StructType: "node",
 			})
+			if err != nil {
+				Warnf("JSON encode failed: %v\n", err)
+			}
 		}
 	} else {
 		printSnapshot = func(sn *restic.Snapshot) {

cmd/restic/cmd_migrate.go

@@ -99,7 +99,7 @@ func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(repo)
+	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err

cmd/restic/cmd_mount.go

@@ -81,7 +81,17 @@ func init() {
 	mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
 }
 
-func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
+func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
+	if opts.SnapshotTemplate == "" {
+		return errors.Fatal("snapshot template string cannot be empty")
+	}
+	if strings.ContainsAny(opts.SnapshotTemplate, `\/`) {
+		return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character")
+	}
+	if len(args) == 0 {
+		return errors.Fatal("wrong number of parameters")
+	}
+
 	debug.Log("start mount")
 	defer debug.Log("finish mount")
 
@@ -91,7 +101,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -103,14 +113,12 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 		return err
 	}
 
-	if _, err := resticfs.Stat(mountpoint); os.IsNotExist(errors.Cause(err)) {
-		Verbosef("Mountpoint %s doesn't exist, creating it\n", mountpoint)
-		err = resticfs.Mkdir(mountpoint, os.ModeDir|0700)
-		if err != nil {
-			return err
-		}
-	}
+	mountpoint := args[0]
 
+	if _, err := resticfs.Stat(mountpoint); os.IsNotExist(errors.Cause(err)) {
+		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
+		return err
+	}
 	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
@@ -125,6 +133,15 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 		}
 	}
 
+	AddCleanupHandler(func() error {
+		debug.Log("running umount cleanup handler for mount at %v", mountpoint)
+		err := umount(mountpoint)
+		if err != nil {
+			Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
+		}
+		return nil
+	})
+
 	c, err := systemFuse.Mount(mountpoint, mountOptions...)
 	if err != nil {
 		return err
@@ -159,30 +176,3 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 func umount(mountpoint string) error {
 	return systemFuse.Unmount(mountpoint)
 }
-
-func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
-	if opts.SnapshotTemplate == "" {
-		return errors.Fatal("snapshot template string cannot be empty")
-	}
-
-	if strings.ContainsAny(opts.SnapshotTemplate, `\/`) {
-		return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character")
-	}
-
-	if len(args) == 0 {
-		return errors.Fatal("wrong number of parameters")
-	}
-
-	mountpoint := args[0]
-
-	AddCleanupHandler(func() error {
-		debug.Log("running umount cleanup handler for mount at %v", mountpoint)
-		err := umount(mountpoint)
-		if err != nil {
-			Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
-		}
-		return nil
-	})
-
-	return mount(opts, gopts, mountpoint)
-}

cmd/restic/cmd_options.go

@@ -23,8 +23,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	DisableAutoGenTag: true,
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Printf("All Extended Options:\n")
+		var maxLen int
 		for _, opt := range options.List() {
-			fmt.Printf("  %-15s   %s\n", opt.Namespace+"."+opt.Name, opt.Text)
+			if l := len(opt.Namespace + "." + opt.Name); l > maxLen {
+				maxLen = l
+			}
+		}
+		for _, opt := range options.List() {
+			fmt.Printf("  %*s  %s\n", -maxLen, opt.Namespace+"."+opt.Name, opt.Text)
 		}
 	},
 }

cmd/restic/cmd_prune.go

@@ -1,15 +1,23 @@
 package main
 
 import (
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 )
 
+var errorIndexIncomplete = errors.Fatal("index is not complete")
+var errorPacksMissing = errors.Fatal("packs from index missing in repo")
+var errorSizeNotMatching = errors.Fatal("pack size does not match calculated size from index")
+
 var cmdPrune = &cobra.Command{
 	Use:   "prune [flags]",
 	Short: "Remove unneeded data from the repository",
@@ -24,12 +32,91 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runPrune(globalOptions)
+		return runPrune(pruneOptions, globalOptions)
 	},
 }
 
+// PruneOptions collects all options for the cleanup command.
+type PruneOptions struct {
+	DryRun bool
+
+	MaxUnused      string
+	maxUnusedBytes func(used uint64) (unused uint64) // calculates the number of unused bytes after repacking, according to MaxUnused
+
+	MaxRepackSize  string
+	MaxRepackBytes uint64
+
+	RepackCachableOnly bool
+}
+
+var pruneOptions PruneOptions
+
 func init() {
 	cmdRoot.AddCommand(cmdPrune)
+	f := cmdPrune.Flags()
+	f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
+	addPruneOptions(cmdPrune)
+}
+
+func addPruneOptions(c *cobra.Command) {
+	f := c.Flags()
+	f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
+	f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
+	f.BoolVar(&pruneOptions.RepackCachableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
+}
+
+func verifyPruneOptions(opts *PruneOptions) error {
+	if len(opts.MaxRepackSize) > 0 {
+		size, err := parseSizeStr(opts.MaxRepackSize)
+		if err != nil {
+			return err
+		}
+		opts.MaxRepackBytes = uint64(size)
+	}
+
+	maxUnused := strings.TrimSpace(opts.MaxUnused)
+	if maxUnused == "" {
+		return errors.Fatalf("invalid value for --max-unused: %q", opts.MaxUnused)
+	}
+
+	// parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes
+	switch {
+	case maxUnused == "unlimited":
+		opts.maxUnusedBytes = func(used uint64) uint64 {
+			return math.MaxUint64
+		}
+
+	case strings.HasSuffix(maxUnused, "%"):
+		maxUnused = strings.TrimSuffix(maxUnused, "%")
+		p, err := strconv.ParseFloat(maxUnused, 64)
+		if err != nil {
+			return errors.Fatalf("invalid percentage %q passed for --max-unused: %v", opts.MaxUnused, err)
+		}
+
+		if p < 0 {
+			return errors.Fatal("percentage for --max-unused must be positive")
+		}
+
+		if p >= 100 {
+			return errors.Fatal("percentage for --max-unused must be below 100%")
+		}
+
+		opts.maxUnusedBytes = func(used uint64) uint64 {
+			return uint64(p / (100 - p) * float64(used))
+		}
+
+	default:
+		size, err := parseSizeStr(maxUnused)
+		if err != nil {
+			return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err)
+		}
+
+		opts.maxUnusedBytes = func(used uint64) uint64 {
+			return uint64(size)
+		}
+	}
+
+	return nil
 }
 
 func shortenStatus(maxLength int, s string) string {
@@ -44,215 +131,415 @@ func shortenStatus(maxLength int, s string) string {
 	return s[:maxLength-3] + "..."
 }
 
-func runPrune(gopts GlobalOptions) error {
+func runPrune(opts PruneOptions, gopts GlobalOptions) error {
+	err := verifyPruneOptions(&opts)
+	if err != nil {
+		return err
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(repo)
+	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
+	return runPruneWithRepo(opts, gopts, repo, restic.NewIDSet())
+}
+
+func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
 	// we do not need index updates while pruning!
 	repo.DisableAutoIndexUpdate()
 
-	return pruneRepository(gopts, repo)
-}
-
-func mixedBlobs(list []restic.Blob) bool {
-	var tree, data bool
-
-	for _, pb := range list {
-		switch pb.Type {
-		case restic.TreeBlob:
-			tree = true
-		case restic.DataBlob:
-			data = true
-		}
-
-		if tree && data {
-			return true
-		}
+	if repo.Cache == nil {
+		Print("warning: running prune without a cache, this may be very slow!\n")
 	}
 
-	return false
+	Verbosef("loading indexes...\n")
+	err := repo.LoadIndex(gopts.ctx)
+	if err != nil {
+		return err
+	}
+
+	usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
+	if err != nil {
+		return err
+	}
+
+	return prune(opts, gopts, repo, usedBlobs)
 }
 
-func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
+type packInfo struct {
+	usedBlobs      uint
+	unusedBlobs    uint
+	duplicateBlobs uint
+	usedSize       uint64
+	unusedSize     uint64
+	tpe            restic.BlobType
+}
+
+type packInfoWithID struct {
+	ID restic.ID
+	packInfo
+}
+
+// prune selects which files to rewrite and then does that. The map usedBlobs is
+// modified in the process.
+func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedBlobs restic.BlobSet) error {
 	ctx := gopts.ctx
 
-	err := repo.LoadIndex(ctx)
-	if err != nil {
-		return err
-	}
-
 	var stats struct {
-		blobs     int
-		packs     int
-		snapshots int
-		bytes     int64
+		blobs struct {
+			used      uint
+			duplicate uint
+			unused    uint
+			remove    uint
+			repack    uint
+			repackrm  uint
+		}
+		size struct {
+			used      uint64
+			duplicate uint64
+			unused    uint64
+			remove    uint64
+			repack    uint64
+			repackrm  uint64
+			unref     uint64
+		}
+		packs struct {
+			used       uint
+			unused     uint
+			partlyUsed uint
+			keep       uint
+		}
 	}
 
-	Verbosef("counting files in repo\n")
-	err = repo.List(ctx, restic.PackFile, func(restic.ID, int64) error {
-		stats.packs++
+	Verbosef("searching used packs...\n")
+
+	keepBlobs := restic.NewBlobSet()
+	duplicateBlobs := restic.NewBlobSet()
+
+	// iterate over all blobs in index to find out which blobs are duplicates
+	for blob := range repo.Index().Each(ctx) {
+		bh := blob.BlobHandle
+		size := uint64(blob.Length)
+		switch {
+		case usedBlobs.Has(bh): // used blob, move to keepBlobs
+			usedBlobs.Delete(bh)
+			keepBlobs.Insert(bh)
+			stats.size.used += size
+			stats.blobs.used++
+		case keepBlobs.Has(bh): // duplicate blob
+			duplicateBlobs.Insert(bh)
+			stats.size.duplicate += size
+			stats.blobs.duplicate++
+		default:
+			stats.size.unused += size
+			stats.blobs.unused++
+		}
+	}
+
+	// Check if all used blobs have been found in index
+	if len(usedBlobs) != 0 {
+		Warnf("%v not found in the index\n\n"+
+			"Integrity check failed: Data seems to be missing.\n"+
+			"Will not start prune to prevent (additional) data loss!\n"+
+			"Please report this error (along with the output of the 'prune' run) at\n"+
+			"https://github.com/restic/restic/issues/new/choose", usedBlobs)
+		return errorIndexIncomplete
+	}
+
+	indexPack := make(map[restic.ID]packInfo)
+
+	// save computed pack header size
+	for pid, hdrSize := range repo.Index().PackSize(ctx, true) {
+		// initialize tpe with NumBlobTypes to indicate it's not set
+		indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
+	}
+
+	// iterate over all blobs in index to generate packInfo
+	for blob := range repo.Index().Each(ctx) {
+		ip := indexPack[blob.PackID]
+
+		// Set blob type if not yet set
+		if ip.tpe == restic.NumBlobTypes {
+			ip.tpe = blob.Type
+		}
+
+		// mark mixed packs with "Invalid blob type"
+		if ip.tpe != blob.Type {
+			ip.tpe = restic.InvalidBlob
+		}
+
+		bh := blob.BlobHandle
+		size := uint64(blob.Length)
+		switch {
+		case duplicateBlobs.Has(bh): // duplicate blob
+			ip.usedSize += size
+			ip.duplicateBlobs++
+		case keepBlobs.Has(bh): // used blob, not duplicate
+			ip.usedSize += size
+			ip.usedBlobs++
+		default: // unused blob
+			ip.unusedSize += size
+			ip.unusedBlobs++
+		}
+		// update indexPack
+		indexPack[blob.PackID] = ip
+	}
+
+	Verbosef("collecting packs for deletion and repacking\n")
+	removePacksFirst := restic.NewIDSet()
+	removePacks := restic.NewIDSet()
+	repackPacks := restic.NewIDSet()
+
+	var repackCandidates []packInfoWithID
+	repackAllPacksWithDuplicates := true
+
+	keep := func(p packInfo) {
+		stats.packs.keep++
+		if p.duplicateBlobs > 0 {
+			repackAllPacksWithDuplicates = false
+		}
+	}
+
+	// loop over all packs and decide what to do
+	bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed")
+	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
+		p, ok := indexPack[id]
+		if !ok {
+			// Pack was not referenced in index and is not used  => immediately remove!
+			Verboseff("will remove pack %v as it is unused and not indexed\n", id.Str())
+			removePacksFirst.Insert(id)
+			stats.size.unref += uint64(packSize)
+			return nil
+		}
+
+		if p.unusedSize+p.usedSize != uint64(packSize) &&
+			!(p.usedBlobs == 0 && p.duplicateBlobs == 0) {
+			// Pack size does not fit and pack is needed => error
+			// If the pack is not needed, this is no error, the pack can
+			// and will be simply removed, see below.
+			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.",
+				id.Str(), p.unusedSize+p.usedSize, packSize)
+			return errorSizeNotMatching
+		}
+
+		// statistics
+		switch {
+		case p.usedBlobs == 0 && p.duplicateBlobs == 0:
+			stats.packs.unused++
+		case p.unusedBlobs == 0:
+			stats.packs.used++
+		default:
+			stats.packs.partlyUsed++
+		}
+
+		// decide what to do
+		switch {
+		case p.usedBlobs == 0 && p.duplicateBlobs == 0:
+			// All blobs in pack are no longer used => remove pack!
+			removePacks.Insert(id)
+			stats.blobs.remove += p.unusedBlobs
+			stats.size.remove += p.unusedSize
+
+		case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
+			// if this is a data pack and --repack-cacheable-only is set => keep pack!
+			keep(p)
+
+		case p.unusedBlobs == 0 && p.duplicateBlobs == 0 && p.tpe != restic.InvalidBlob:
+			// All blobs in pack are used and not duplicates/mixed => keep pack!
+			keep(p)
+
+		default:
+			// all other packs are candidates for repacking
+			repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p})
+		}
+
+		delete(indexPack, id)
+		bar.Add(1)
 		return nil
 	})
+	bar.Done()
 	if err != nil {
 		return err
 	}
 
-	Verbosef("building new index for repo\n")
+	// At this point indexPacks contains only missing packs!
 
-	bar := newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
-	idx, invalidFiles, err := index.New(ctx, repo, restic.NewIDSet(), bar)
-	if err != nil {
-		return err
+	// missing packs that are not needed can be ignored
+	ignorePacks := restic.NewIDSet()
+	for id, p := range indexPack {
+		if p.usedBlobs == 0 && p.duplicateBlobs == 0 {
+			ignorePacks.Insert(id)
+			stats.blobs.remove += p.unusedBlobs
+			stats.size.remove += p.unusedSize
+			delete(indexPack, id)
+		}
 	}
 
-	for _, id := range invalidFiles {
-		Warnf("incomplete pack file (will be removed): %v\n", id)
+	if len(indexPack) != 0 {
+		Warnf("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
+		for id := range indexPack {
+			Warnf("  %v\n", id)
+		}
+		return errorPacksMissing
+	}
+	if len(ignorePacks) != 0 {
+		Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
+		for id := range ignorePacks {
+			Warnf("will forget missing pack file %v\n", id)
+		}
 	}
 
-	blobs := 0
-	for _, pack := range idx.Packs {
-		stats.bytes += pack.Size
-		blobs += len(pack.Entries)
+	// calculate limit for number of unused bytes in the repo after repacking
+	maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
+
+	// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
+	// This is equivalent to sorting by unused / total space.
+	// Instead of unused[i] / used[i] > unused[j] / used[j] we use
+	// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
+	// Morover duplicates and packs containing trees are sorted to the beginning
+	sort.Slice(repackCandidates, func(i, j int) bool {
+		pi := repackCandidates[i].packInfo
+		pj := repackCandidates[j].packInfo
+		switch {
+		case pi.duplicateBlobs > 0 && pj.duplicateBlobs == 0:
+			return true
+		case pj.duplicateBlobs > 0 && pi.duplicateBlobs == 0:
+			return false
+		case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
+			return true
+		case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
+			return false
+		}
+		return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
+	})
+
+	repack := func(id restic.ID, p packInfo) {
+		repackPacks.Insert(id)
+		stats.blobs.repack += p.unusedBlobs + p.duplicateBlobs + p.usedBlobs
+		stats.size.repack += p.unusedSize + p.usedSize
+		stats.blobs.repackrm += p.unusedBlobs
+		stats.size.repackrm += p.unusedSize
 	}
-	Verbosef("repository contains %v packs (%v blobs) with %v\n",
-		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
-	blobCount := make(map[restic.BlobHandle]int)
-	var duplicateBlobs uint64
-	var duplicateBytes uint64
+	for _, p := range repackCandidates {
+		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
 
-	// find duplicate blobs
-	for _, p := range idx.Packs {
-		for _, entry := range p.Entries {
-			stats.blobs++
-			h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
-			blobCount[h]++
+		reachedRepackSize := false
+		if opts.MaxRepackBytes > 0 {
+			reachedRepackSize = stats.size.repack+p.unusedSize+p.usedSize > opts.MaxRepackBytes
+		}
 
-			if blobCount[h] > 1 {
-				duplicateBlobs++
-				duplicateBytes += uint64(entry.Length)
+		switch {
+		case reachedRepackSize:
+			keep(p.packInfo)
+
+		case p.duplicateBlobs > 0, p.tpe != restic.DataBlob:
+			// repacking duplicates/non-data is only limited by repackSize
+			repack(p.ID, p.packInfo)
+
+		case reachedUnusedSizeAfter:
+			// for all other packs stop repacking if tolerated unused size is reached.
+			keep(p.packInfo)
+
+		default:
+			repack(p.ID, p.packInfo)
+		}
+	}
+
+	// if all duplicates are repacked, print out correct statistics
+	if repackAllPacksWithDuplicates {
+		stats.blobs.repackrm += stats.blobs.duplicate
+		stats.size.repackrm += stats.size.duplicate
+	}
+
+	Verboseff("\nused:        %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
+	if stats.blobs.duplicate > 0 {
+		Verboseff("duplicates:  %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
+	}
+	Verboseff("unused:      %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
+	if stats.size.unref > 0 {
+		Verboseff("unreferenced:                   %s\n", formatBytes(stats.size.unref))
+	}
+	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
+	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
+	unusedSize := stats.size.duplicate + stats.size.unused
+	Verboseff("total:       %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
+	Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
+
+	Verbosef("\nto repack:   %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
+	Verbosef("this removes %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
+	Verbosef("to delete:   %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
+	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
+	Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
+	Verbosef("remaining:   %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
+	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
+	Verbosef("unused size after prune: %s (%s of remaining size)\n",
+		formatBytes(unusedAfter), formatPercent(unusedAfter, totalSize-totalPruneSize))
+	Verbosef("\n")
+	Verboseff("totally used packs: %10d\n", stats.packs.used)
+	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
+	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
+
+	Verboseff("to keep:   %10d packs\n", stats.packs.keep)
+	Verboseff("to repack: %10d packs\n", len(repackPacks))
+	Verboseff("to delete: %10d packs\n", len(removePacks))
+	if len(removePacksFirst) > 0 {
+		Verboseff("to delete: %10d unreferenced packs\n\n", len(removePacksFirst))
+	}
+
+	if opts.DryRun {
+		if !gopts.JSON && gopts.verbosity >= 2 {
+			if len(removePacksFirst) > 0 {
+				Printf("Would have removed the following unreferenced packs:\n%v\n\n", removePacksFirst)
 			}
+			Printf("Would have repacked and removed the following packs:\n%v\n\n", repackPacks)
+			Printf("Would have removed the following no longer used packs:\n%v\n\n", removePacks)
 		}
+		// Always quit here if DryRun was set!
+		return nil
 	}
 
-	Verbosef("processed %d blobs: %d duplicate blobs, %v duplicate\n",
-		stats.blobs, duplicateBlobs, formatBytes(uint64(duplicateBytes)))
-	Verbosef("load all snapshots\n")
-
-	// find referenced blobs
-	snapshots, err := restic.LoadAllSnapshots(ctx, repo)
-	if err != nil {
-		return err
+	// unreferenced packs can be safely deleted first
+	if len(removePacksFirst) != 0 {
+		Verbosef("deleting unreferenced packs\n")
+		DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile)
 	}
 
-	stats.snapshots = len(snapshots)
-
-	usedBlobs, err := getUsedBlobs(gopts, repo, snapshots)
-	if err != nil {
-		return err
-	}
-
-	var missingBlobs []restic.BlobHandle
-	for h := range usedBlobs {
-		if _, ok := blobCount[h]; !ok {
-			missingBlobs = append(missingBlobs, h)
-		}
-	}
-	if len(missingBlobs) > 0 {
-		return errors.Fatalf("%v not found in the new index\n"+
-			"Data blobs seem to be missing, aborting prune to prevent further data loss!\n"+
-			"Please report this error (along with the output of the 'prune' run) at\n"+
-			"https://github.com/restic/restic/issues/new/choose", missingBlobs)
-	}
-
-	Verbosef("found %d of %d data blobs still in use, removing %d blobs\n",
-		len(usedBlobs), stats.blobs, stats.blobs-len(usedBlobs))
-
-	// find packs that need a rewrite
-	rewritePacks := restic.NewIDSet()
-	for _, pack := range idx.Packs {
-		if mixedBlobs(pack.Entries) {
-			rewritePacks.Insert(pack.ID)
-			continue
-		}
-
-		for _, blob := range pack.Entries {
-			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-			if !usedBlobs.Has(h) {
-				rewritePacks.Insert(pack.ID)
-				continue
-			}
-
-			if blobCount[h] > 1 {
-				rewritePacks.Insert(pack.ID)
-			}
-		}
-	}
-
-	removeBytes := duplicateBytes
-
-	// find packs that are unneeded
-	removePacks := restic.NewIDSet()
-
-	Verbosef("will remove %d invalid files\n", len(invalidFiles))
-	for _, id := range invalidFiles {
-		removePacks.Insert(id)
-	}
-
-	for packID, p := range idx.Packs {
-
-		hasActiveBlob := false
-		for _, blob := range p.Entries {
-			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-			if usedBlobs.Has(h) {
-				hasActiveBlob = true
-				continue
-			}
-
-			removeBytes += uint64(blob.Length)
-		}
-
-		if hasActiveBlob {
-			continue
-		}
-
-		removePacks.Insert(packID)
-
-		if !rewritePacks.Has(packID) {
-			return errors.Fatalf("pack %v is unneeded, but not contained in rewritePacks", packID.Str())
-		}
-
-		rewritePacks.Delete(packID)
-	}
-
-	Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
-		len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
-
-	var obsoletePacks restic.IDSet
-	if len(rewritePacks) != 0 {
-		bar := newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewritten")
-		obsoletePacks, err = repository.Repack(ctx, repo, rewritePacks, usedBlobs, bar)
+	if len(repackPacks) != 0 {
+		Verbosef("repacking packs\n")
+		bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked")
+		_, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar)
+		bar.Done()
 		if err != nil {
-			return err
+			return errors.Fatalf("%s", err)
 		}
+
+		// Also remove repacked packs
+		removePacks.Merge(repackPacks)
 	}
 
-	removePacks.Merge(obsoletePacks)
+	if len(ignorePacks) == 0 {
+		ignorePacks = removePacks
+	} else {
+		ignorePacks.Merge(removePacks)
+	}
 
-	if err = rebuildIndex(ctx, repo, removePacks); err != nil {
-		return err
+	if len(ignorePacks) != 0 {
+		err = rebuildIndexFiles(gopts, repo, ignorePacks, nil)
+		if err != nil {
+			return errors.Fatalf("%s", err)
+		}
 	}
 
 	if len(removePacks) != 0 {
-		Verbosef("remove %d old packs\n", len(removePacks))
+		Verbosef("removing %d old packs\n", len(removePacks))
 		DeleteFiles(gopts, repo, removePacks, restic.PackFile)
 	}
 
@@ -260,30 +547,54 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	return nil
 }
 
-func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, snapshots []*restic.Snapshot) (usedBlobs restic.BlobSet, err error) {
+func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
+	Verbosef("rebuilding index\n")
+
+	idx := (repo.Index()).(*repository.MasterIndex)
+	packcount := uint64(len(idx.Packs(removePacks)))
+	bar := newProgressMax(!gopts.Quiet, packcount, "packs processed")
+	obsoleteIndexes, err := idx.Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
+	bar.Done()
+	if err != nil {
+		return err
+	}
+
+	Verbosef("deleting obsolete index files\n")
+	return DeleteFilesChecked(gopts, repo, obsoleteIndexes, restic.IndexFile)
+}
+
+func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) {
 	ctx := gopts.ctx
 
-	Verbosef("find data that is still in use for %d snapshots\n", len(snapshots))
+	var snapshotTrees restic.IDs
+	Verbosef("loading all snapshots...\n")
+	err = restic.ForAllSnapshots(gopts.ctx, repo, ignoreSnapshots,
+		func(id restic.ID, sn *restic.Snapshot, err error) error {
+			debug.Log("add snapshot %v (tree %v, error %v)", id, *sn.Tree, err)
+			if err != nil {
+				return err
+			}
+			snapshotTrees = append(snapshotTrees, *sn.Tree)
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+
+	Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
 
 	usedBlobs = restic.NewBlobSet()
 
-	bar := newProgressMax(!gopts.Quiet, uint64(len(snapshots)), "snapshots")
-	bar.Start()
+	bar := newProgressMax(!gopts.Quiet, uint64(len(snapshotTrees)), "snapshots")
 	defer bar.Done()
-	for _, sn := range snapshots {
-		debug.Log("process snapshot %v", sn.ID())
 
-		err = restic.FindUsedBlobs(ctx, repo, *sn.Tree, usedBlobs)
-		if err != nil {
-			if repo.Backend().IsNotExist(err) {
-				return nil, errors.Fatal("unable to load a tree from the repo: " + err.Error())
-			}
-
-			return nil, err
+	err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
+	if err != nil {
+		if repo.Backend().IsNotExist(err) {
+			return nil, errors.Fatal("unable to load a tree from the repo: " + err.Error())
 		}
 
-		debug.Log("processed snapshot %v", sn.ID())
-		bar.Report(restic.Stat{Blobs: 1})
+		return nil, err
 	}
 	return usedBlobs, nil
 }

cmd/restic/cmd_rebuild_index.go

@@ -1,10 +1,7 @@
 package main
 
 import (
-	"context"
-
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
@@ -12,7 +9,7 @@ import (
 
 var cmdRebuildIndex = &cobra.Command{
 	Use:   "rebuild-index [flags]",
-	Short: "Build a new index file",
+	Short: "Build a new index",
 	Long: `
 The "rebuild-index" command creates a new index based on the pack files in the
 repository.
@@ -24,78 +21,110 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRebuildIndex(globalOptions)
+		return runRebuildIndex(rebuildIndexOptions, globalOptions)
 	},
 }
 
-func init() {
-	cmdRoot.AddCommand(cmdRebuildIndex)
+// RebuildIndexOptions collects all options for the rebuild-index command.
+type RebuildIndexOptions struct {
+	ReadAllPacks bool
 }
 
-func runRebuildIndex(gopts GlobalOptions) error {
+var rebuildIndexOptions RebuildIndexOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdRebuildIndex)
+	f := cmdRebuildIndex.Flags()
+	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
+
+}
+
+func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
 	}
 
-	lock, err := lockRepoExclusive(repo)
+	lock, err := lockRepoExclusive(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-	return rebuildIndex(ctx, repo, restic.NewIDSet())
+	return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
 }
 
-func rebuildIndex(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet) error {
-	Verbosef("counting files in repo\n")
+func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
+	ctx := gopts.ctx
 
-	var packs uint64
-	err := repo.List(ctx, restic.PackFile, func(restic.ID, int64) error {
-		packs++
+	var obsoleteIndexes restic.IDs
+	packSizeFromList := make(map[restic.ID]int64)
+	packSizeFromIndex := make(map[restic.ID]int64)
+	removePacks := restic.NewIDSet()
+
+	if opts.ReadAllPacks {
+		// get list of old index files but start with empty index
+		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
+			obsoleteIndexes = append(obsoleteIndexes, id)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		Verbosef("loading indexes...\n")
+		err := repo.LoadIndex(gopts.ctx)
+		if err != nil {
+			return err
+		}
+		packSizeFromIndex = repo.Index().PackSize(ctx, false)
+	}
+
+	Verbosef("getting pack files to read...\n")
+	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
+		size, ok := packSizeFromIndex[id]
+		if !ok || size != packSize {
+			// Pack was not referenced in index or size does not match
+			packSizeFromList[id] = packSize
+			removePacks.Insert(id)
+		}
+		if !ok {
+			Warnf("adding pack file to index %v\n", id)
+		} else if size != packSize {
+			Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
+		}
+		delete(packSizeFromIndex, id)
 		return nil
 	})
 	if err != nil {
 		return err
 	}
-
-	bar := newProgressMax(!globalOptions.Quiet, packs-uint64(len(ignorePacks)), "packs")
-	idx, invalidFiles, err := index.New(ctx, repo, ignorePacks, bar)
-	if err != nil {
-		return err
+	for id := range packSizeFromIndex {
+		// forget pack files that are referenced in the index but do not exist
+		// when rebuilding the index
+		removePacks.Insert(id)
+		Warnf("removing not found pack file %v\n", id)
 	}
 
-	if globalOptions.verbosity >= 2 {
+	if len(packSizeFromList) > 0 {
+		Verbosef("reading pack files\n")
+		bar := newProgressMax(!globalOptions.Quiet, uint64(len(packSizeFromList)), "packs")
+		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
+		bar.Done()
+		if err != nil {
+			return err
+		}
+
 		for _, id := range invalidFiles {
-			Printf("skipped incomplete pack file: %v\n", id)
+			Verboseff("skipped incomplete pack file: %v\n", id)
 		}
 	}
 
-	Verbosef("finding old index files\n")
-
-	var supersedes restic.IDs
-	err = repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
-		supersedes = append(supersedes, id)
-		return nil
-	})
+	err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
 	if err != nil {
 		return err
 	}
-
-	ids, err := idx.Save(ctx, repo, supersedes)
-	if err != nil {
-		return errors.Fatalf("unable to save index, last error was: %v", err)
-	}
-
-	Verbosef("saved new indexes as %v\n", ids)
-
-	Verbosef("remove %d old index files\n", len(supersedes))
-	err = DeleteFilesChecked(globalOptions, repo, restic.NewIDSet(supersedes...), restic.IndexFile)
-	if err != nil {
-		return errors.Fatalf("unable to remove an old index: %v\n", err)
-	}
+	Verbosef("done\n")
 
 	return nil
 }

cmd/restic/cmd_recover.go

@@ -13,7 +13,7 @@ var cmdRecover = &cobra.Command{
 	Use:   "recover [flags]",
 	Short: "Recover data from the repository",
 	Long: `
-The "recover" command build a new snapshot from all directories it can find in
+The "recover" command builds a new snapshot from all directories it can find in
 the raw data of the repository. It can be used if, for example, a snapshot has
 been removed by accident with "forget".
 
@@ -43,7 +43,7 @@ func runRecover(gopts GlobalOptions) error {
 		return err
 	}
 
-	lock, err := lockRepo(repo)
+	lock, err := lockRepo(gopts.ctx, repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
@@ -117,7 +117,10 @@ func runRecover(gopts GlobalOptions) error {
 			ModTime:    time.Now(),
 			ChangeTime: time.Now(),
 		}
-		tree.Insert(&node)
+		err = tree.Insert(&node)
+		if err != nil {
+			return err
+		}
 	}
 
 	treeID, err := repo.SaveTree(gopts.ctx, tree)

cmd/restic/cmd_restore.go

@@ -102,7 +102,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -122,13 +122,13 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
-		id, err = restic.FindSnapshot(repo, snapshotIDString)
+		id, err = restic.FindSnapshot(ctx, repo, snapshotIDString)
 		if err != nil {
 			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
 		}
 	}
 
-	res, err := restorer.NewRestorer(repo, id)
+	res, err := restorer.NewRestorer(ctx, repo, id)
 	if err != nil {
 		Exitf(2, "creating restorer failed: %v\n", err)
 	}
@@ -140,13 +140,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		return nil
 	}
 
+	excludePatterns := filter.ParsePatterns(opts.Exclude)
+	insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)
 	selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
-		matched, _, err := filter.List(opts.Exclude, item)
+		matched, err := filter.List(excludePatterns, item)
 		if err != nil {
 			Warnf("error for exclude pattern: %v", err)
 		}
 
-		matchedInsensitive, _, err := filter.List(opts.InsensitiveExclude, strings.ToLower(item))
+		matchedInsensitive, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item))
 		if err != nil {
 			Warnf("error for iexclude pattern: %v", err)
 		}
@@ -161,13 +163,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		return selectedForRestore, childMayBeSelected
 	}
 
+	includePatterns := filter.ParsePatterns(opts.Include)
+	insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude)
 	selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
-		matched, childMayMatch, err := filter.List(opts.Include, item)
+		matched, childMayMatch, err := filter.ListWithChild(includePatterns, item)
 		if err != nil {
 			Warnf("error for include pattern: %v", err)
 		}
 
-		matchedInsensitive, childMayMatchInsensitive, err := filter.List(opts.InsensitiveInclude, strings.ToLower(item))
+		matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item))
 		if err != nil {
 			Warnf("error for iexclude pattern: %v", err)
 		}
@@ -187,14 +191,26 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target)
 
 	err = res.RestoreTo(ctx, opts.Target)
-	if err == nil && opts.Verify {
+	if err != nil {
+		return err
+	}
+
+	if totalErrors > 0 {
+		return errors.Fatalf("There were %d errors\n", totalErrors)
+	}
+
+	if opts.Verify {
 		Verbosef("verifying files in %s\n", opts.Target)
 		var count int
 		count, err = res.VerifyFiles(ctx, opts.Target)
+		if err != nil {
+			return err
+		}
+		if totalErrors > 0 {
+			return errors.Fatalf("There were %d errors\n", totalErrors)
+		}
 		Verbosef("finished verifying %d files in %s\n", count, opts.Target)
 	}
-	if totalErrors > 0 {
-		Printf("There were %d errors\n", totalErrors)
-	}
-	return err
+
+	return nil
 }

cmd/restic/cmd_self_update.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/selfupdate"
@@ -56,11 +57,18 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
 
 	fi, err := os.Lstat(opts.Output)
 	if err != nil {
-		return err
-	}
-
-	if !fi.Mode().IsRegular() {
-		return errors.Errorf("output file %v is not a normal file, use --output to specify a different file", opts.Output)
+		dirname := filepath.Dir(opts.Output)
+		di, err := os.Lstat(dirname)
+		if err != nil {
+			return err
+		}
+		if !di.Mode().IsDir() {
+			return errors.Fatalf("output parent path %v is not a directory, use --output to specify a different file path", dirname)
+		}
+	} else {
+		if !fi.Mode().IsRegular() {
+			return errors.Fatalf("output path %v is not a normal file, use --output to specify a different file path", opts.Output)
+		}
 	}
 
 	Printf("writing restic to %v\n", opts.Output)

cmd/restic/cmd_snapshots.go

@@ -47,9 +47,9 @@ func init() {
 
 	f := cmdSnapshots.Flags()
 	f.StringArrayVarP(&snapshotOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host` (can be specified multiple times)")
-	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
+	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
 	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
-	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
+	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format")
 	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
 	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
 }
@@ -61,7 +61,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -243,7 +243,10 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 		}
 	}
 
-	tab.Write(stdout)
+	err := tab.Write(stdout)
+	if err != nil {
+		Warnf("error printing: %v\n", err)
+	}
 }
 
 // PrintSnapshotGroupHeader prints which group of the group-by option the
@@ -298,7 +301,7 @@ type SnapshotGroup struct {
 // printSnapshotsJSON writes the JSON representation of list to stdout.
 func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {
 	if grouped {
-		var snapshotGroups []SnapshotGroup
+		snapshotGroups := []SnapshotGroup{}
 
 		for k, list := range snGroups {
 			var key restic.SnapshotGroupKey
@@ -330,7 +333,7 @@ func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapsho
 	}
 
 	// Old behavior
-	var snapshots []Snapshot
+	snapshots := []Snapshot{}
 
 	for _, list := range snGroups {
 		for _, sn := range list {

cmd/restic/cmd_snapshots_test.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+// Regression test for #2979: no snapshots should print as [], not null.
+func TestEmptySnapshotGroupJSON(t *testing.T) {
+	for _, grouped := range []bool{false, true} {
+		var w strings.Builder
+		err := printSnapshotGroupJSON(&w, nil, grouped)
+		rtest.OK(t, err)
+
+		rtest.Equals(t, "[]", strings.TrimSpace(w.String()))
+	}
+}

cmd/restic/cmd_stats.go

@@ -91,7 +91,7 @@ func runStats(gopts GlobalOptions, args []string) error {
 	}
 
 	if !gopts.NoLock {
-		lock, err := lockRepo(repo)
+		lock, err := lockRepo(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -166,7 +166,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 	if statsOptions.countMode == countModeRawData {
 		// count just the sizes of unique blobs; we don't need to walk the tree
 		// ourselves in this case, since a nifty function does it for us
-		return restic.FindUsedBlobs(ctx, repo, *snapshot.Tree, stats.blobs)
+		return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
 	}
 
 	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats))

cmd/restic/cmd_tag.go

@@ -38,9 +38,9 @@ type TagOptions struct {
 	Hosts      []string
 	Paths      []string
 	Tags       restic.TagLists
-	SetTags    []string
-	AddTags    []string
-	RemoveTags []string
+	SetTags    restic.TagLists
+	AddTags    restic.TagLists
+	RemoveTags restic.TagLists
 }
 
 var tagOptions TagOptions
@@ -49,9 +49,9 @@ func init() {
 	cmdRoot.AddCommand(cmdTag)
 
 	tagFlags := cmdTag.Flags()
-	tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
-	tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
-	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
+	tagFlags.Var(&tagOptions.SetTags, "set", "`tags` which will replace the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
+	tagFlags.Var(&tagOptions.AddTags, "add", "`tags` which will be added to the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
+	tagFlags.Var(&tagOptions.RemoveTags, "remove", "`tags` which will be removed from the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
 
 	tagFlags.StringArrayVarP(&tagOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
 	tagFlags.Var(&tagOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
@@ -119,7 +119,7 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		lock, err := lockRepoExclusive(repo)
+		lock, err := lockRepoExclusive(gopts.ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
@@ -130,7 +130,7 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
-		changed, err := changeTags(ctx, repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags)
+		changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
 		if err != nil {
 			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)
 			continue

cmd/restic/delete.go

@@ -9,7 +9,7 @@ import (
 // DeleteFiles deletes the given fileList of fileType in parallel
 // it will print a warning if there is an error, but continue deleting the remaining files
 func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
-	deleteFiles(gopts, true, repo, fileList, fileType)
+	_ = deleteFiles(gopts, true, repo, fileList, fileType)
 }
 
 // DeleteFilesChecked deletes the given fileList of fileType in parallel
@@ -33,8 +33,8 @@ func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository,
 	}()
 
 	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted")
+	defer bar.Done()
 	wg, ctx := errgroup.WithContext(gopts.ctx)
-	bar.Start()
 	for i := 0; i < numDeleteWorkers; i++ {
 		wg.Go(func() error {
 			for id := range fileChan {
@@ -48,15 +48,14 @@ func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository,
 						return err
 					}
 				}
-				if !gopts.JSON && gopts.verbosity >= 2 {
+				if !gopts.JSON && gopts.verbosity > 2 {
 					Verbosef("removed %v\n", h)
 				}
-				bar.Report(restic.Stat{Blobs: 1})
+				bar.Add(1)
 			}
 			return nil
 		})
 	}
 	err := wg.Wait()
-	bar.Done()
 	return err
 }

cmd/restic/exclude.go

@@ -74,8 +74,9 @@ type RejectFunc func(path string, fi os.FileInfo) bool
 // rejectByPattern returns a RejectByNameFunc which rejects files that match
 // one of the patterns.
 func rejectByPattern(patterns []string) RejectByNameFunc {
+	parsedPatterns := filter.ParsePatterns(patterns)
 	return func(item string) bool {
-		matched, _, err := filter.List(patterns, item)
+		matched, err := filter.List(parsedPatterns, item)
 		if err != nil {
 			Warnf("error for exclude pattern: %v", err)
 		}
@@ -179,7 +180,9 @@ func isDirExcludedByFile(dir, tagFilename, header string) bool {
 		Warnf("could not open exclusion tagfile: %v", err)
 		return false
 	}
-	defer f.Close()
+	defer func() {
+		_ = f.Close()
+	}()
 	buf := make([]byte, len(header))
 	_, err = io.ReadFull(f, buf)
 	// EOF is handled with a dedicated message, otherwise the warning were too cryptic
@@ -198,12 +201,17 @@ func isDirExcludedByFile(dir, tagFilename, header string) bool {
 	return true
 }
 
-// gatherDevices returns the set of unique device ids of the files and/or
-// directory paths listed in "items".
-func gatherDevices(items []string) (deviceMap map[string]uint64, err error) {
-	deviceMap = make(map[string]uint64)
-	for _, item := range items {
-		item, err = filepath.Abs(filepath.Clean(item))
+// DeviceMap is used to track allowed source devices for backup. This is used to
+// check for crossing mount points during backup (for --one-file-system). It
+// maps the name of a source path to its device ID.
+type DeviceMap map[string]uint64
+
+// NewDeviceMap creates a new device map from the list of source paths.
+func NewDeviceMap(allowedSourcePaths []string) (DeviceMap, error) {
+	deviceMap := make(map[string]uint64)
+
+	for _, item := range allowedSourcePaths {
+		item, err := filepath.Abs(filepath.Clean(item))
 		if err != nil {
 			return nil, err
 		}
@@ -212,34 +220,63 @@ func gatherDevices(items []string) (deviceMap map[string]uint64, err error) {
 		if err != nil {
 			return nil, err
 		}
+
 		id, err := fs.DeviceID(fi)
 		if err != nil {
 			return nil, err
 		}
+
 		deviceMap[item] = id
 	}
+
 	if len(deviceMap) == 0 {
 		return nil, errors.New("zero allowed devices")
 	}
+
 	return deviceMap, nil
 }
 
+// IsAllowed returns true if the path is located on an allowed device.
+func (m DeviceMap) IsAllowed(item string, deviceID uint64) (bool, error) {
+	for dir := item; ; dir = filepath.Dir(dir) {
+		debug.Log("item %v, test dir %v", item, dir)
+
+		// find a parent directory that is on an allowed device (otherwise
+		// we would not traverse the directory at all)
+		allowedID, ok := m[dir]
+		if !ok {
+			if dir == filepath.Dir(dir) {
+				// arrived at root, no allowed device found. this should not happen.
+				break
+			}
+			continue
+		}
+
+		// if the item has a different device ID than the parent directory,
+		// we crossed a file system boundary
+		if allowedID != deviceID {
+			debug.Log("item %v (dir %v) on disallowed device %d", item, dir, deviceID)
+			return false, nil
+		}
+
+		// item is on allowed device, accept it
+		debug.Log("item %v allowed", item)
+		return true, nil
+	}
+
+	return false, fmt.Errorf("item %v (device ID %v) not found, deviceMap: %v", item, deviceID, m)
+}
+
 // rejectByDevice returns a RejectFunc that rejects files which are on a
 // different file systems than the files/dirs in samples.
 func rejectByDevice(samples []string) (RejectFunc, error) {
-	allowed, err := gatherDevices(samples)
+	deviceMap, err := NewDeviceMap(samples)
 	if err != nil {
 		return nil, err
 	}
-	debug.Log("allowed devices: %v\n", allowed)
+	debug.Log("allowed devices: %v\n", deviceMap)
 
 	return func(item string, fi os.FileInfo) bool {
-		if fi == nil {
-			return false
-		}
-
-		item = filepath.Clean(item)
-
 		id, err := fs.DeviceID(fi)
 		if err != nil {
 			// This should never happen because gatherDevices() would have
@@ -247,26 +284,55 @@ func rejectByDevice(samples []string) (RejectFunc, error) {
 			panic(err)
 		}
 
-		for dir := item; ; dir = filepath.Dir(dir) {
-			debug.Log("item %v, test dir %v", item, dir)
-
-			allowedID, ok := allowed[dir]
-			if !ok {
-				if dir == filepath.Dir(dir) {
-					break
-				}
-				continue
-			}
-
-			if allowedID != id {
-				debug.Log("path %q on disallowed device %d", item, id)
-				return true
-			}
+		allowed, err := deviceMap.IsAllowed(filepath.Clean(item), id)
+		if err != nil {
+			// this should not happen
+			panic(fmt.Sprintf("error checking device ID of %v: %v", item, err))
+		}
 
+		if allowed {
+			// accept item
 			return false
 		}
 
-		panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowed))
+		// reject everything except directories
+		if !fi.IsDir() {
+			return true
+		}
+
+		// special case: make sure we keep mountpoints (directories which
+		// contain a mounted file system). Test this by checking if the parent
+		// directory would be included.
+		parentDir := filepath.Dir(filepath.Clean(item))
+
+		parentFI, err := fs.Lstat(parentDir)
+		if err != nil {
+			debug.Log("item %v: error running lstat() on parent directory: %v", item, err)
+			// if in doubt, reject
+			return true
+		}
+
+		parentDeviceID, err := fs.DeviceID(parentFI)
+		if err != nil {
+			debug.Log("item %v: getting device ID of parent directory: %v", item, err)
+			// if in doubt, reject
+			return true
+		}
+
+		parentAllowed, err := deviceMap.IsAllowed(parentDir, parentDeviceID)
+		if err != nil {
+			debug.Log("item %v: error checking parent directory: %v", item, err)
+			// if in doubt, reject
+			return true
+		}
+
+		if parentAllowed {
+			// we found a mount point, so accept the directory
+			return false
+		}
+
+		// reject everything else
+		return true
 	}, nil
 }
 
@@ -317,6 +383,10 @@ func rejectBySize(maxSizeStr string) (RejectFunc, error) {
 }
 
 func parseSizeStr(sizeStr string) (int64, error) {
+	if sizeStr == "" {
+		return 0, errors.New("expected size, got empty string")
+	}
+
 	numStr := sizeStr[:len(sizeStr)-1]
 	var unit int64 = 1
 
@@ -336,7 +406,7 @@ func parseSizeStr(sizeStr string) (int64, error) {
 	}
 	value, err := strconv.ParseInt(numStr, 10, 64)
 	if err != nil {
-		return 0, nil
+		return 0, err
 	}
 	return value * unit, nil
 }

cmd/restic/exclude_test.go

@@ -219,6 +219,25 @@ func TestParseSizeStr(t *testing.T) {
 	}
 }
 
+func TestParseInvalidSizeStr(t *testing.T) {
+	invalidSizes := []string{
+		"",
+		" ",
+		"foobar",
+		"zzz",
+	}
+
+	for _, s := range invalidSizes {
+		v, err := parseSizeStr(s)
+		if err == nil {
+			t.Errorf("wanted error for invalid value %q, got nil", s)
+		}
+		if v != 0 {
+			t.Errorf("wanted zero for invalid value %q, got: %v", s, v)
+		}
+	}
+}
+
 // TestIsExcludedByFileSize is for testing the instance of
 // --exclude-larger-than parameters
 func TestIsExcludedByFileSize(t *testing.T) {
@@ -299,3 +318,47 @@ func TestIsExcludedByFileSize(t *testing.T) {
 		}
 	}
 }
+
+func TestDeviceMap(t *testing.T) {
+	deviceMap := DeviceMap{
+		filepath.FromSlash("/"):          1,
+		filepath.FromSlash("/usr/local"): 5,
+	}
+
+	var tests = []struct {
+		item     string
+		deviceID uint64
+		allowed  bool
+	}{
+		{"/root", 1, true},
+		{"/usr", 1, true},
+
+		{"/proc", 2, false},
+		{"/proc/1234", 2, false},
+
+		{"/usr", 3, false},
+		{"/usr/share", 3, false},
+
+		{"/usr/local", 5, true},
+		{"/usr/local/foobar", 5, true},
+
+		{"/usr/local/foobar/submount", 23, false},
+		{"/usr/local/foobar/submount/file", 23, false},
+
+		{"/usr/local/foobar/outhersubmount", 1, false},
+		{"/usr/local/foobar/outhersubmount/otherfile", 1, false},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			res, err := deviceMap.IsAllowed(filepath.FromSlash(test.item), test.deviceID)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if res != test.allowed {
+				t.Fatalf("wrong result returned by IsAllowed(%v): want %v, got %v", test.item, test.allowed, res)
+			}
+		})
+	}
+}

cmd/restic/find.go

@@ -29,7 +29,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 						continue
 					}
 				} else {
-					id, err = restic.FindSnapshot(repo, s)
+					id, err = restic.FindSnapshot(ctx, repo, s)
 					if err != nil {
 						Warnf("Ignoring %q, it is not a snapshot id\n", s)
 						continue

cmd/restic/global.go

@@ -39,7 +39,7 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-var version = "0.10.0"
+var version = "0.12.0"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -49,6 +49,7 @@ type backendWrapper func(r restic.Backend) (restic.Backend, error)
 // GlobalOptions hold all global options for restic.
 type GlobalOptions struct {
 	Repo            string
+	RepositoryFile  string
 	PasswordFile    string
 	PasswordCommand string
 	KeyHint         string
@@ -70,7 +71,7 @@ type GlobalOptions struct {
 	stdout   io.Writer
 	stderr   io.Writer
 
-	backendTestHook backendWrapper
+	backendTestHook, backendInnerTestHook backendWrapper
 
 	// verbosity is set as follows:
 	//  0 means: don't print any messages except errors, this is used when --quiet is specified
@@ -101,12 +102,13 @@ func init() {
 
 	f := cmdRoot.PersistentFlags()
 	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
+	f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", os.Getenv("RESTIC_REPOSITORY_FILE"), "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
 	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", os.Getenv("RESTIC_PASSWORD_FILE"), "`file` to read the repository password from (default: $RESTIC_PASSWORD_FILE)")
 	f.StringVarP(&globalOptions.KeyHint, "key-hint", "", os.Getenv("RESTIC_KEY_HINT"), "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)")
 	f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", os.Getenv("RESTIC_PASSWORD_COMMAND"), "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)")
 	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
-	f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify --verbose multiple times or level --verbose=`n`)")
-	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
+	f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=`n`, max level/times is 3)")
+	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories")
 	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
 	f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)")
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
@@ -229,6 +231,13 @@ func Verbosef(format string, args ...interface{}) {
 	}
 }
 
+// Verboseff calls Printf to write the message when the verbosity is >= 2
+func Verboseff(format string, args ...interface{}) {
+	if globalOptions.verbosity >= 2 {
+		Printf(format, args...)
+	}
+}
+
 // PrintProgress wraps fmt.Printf to handle the difference in writing progress
 // information to terminals and non-terminal stdout
 func PrintProgress(format string, args ...interface{}) {
@@ -347,7 +356,7 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
 		password, err = readPasswordTerminal(os.Stdin, os.Stderr, prompt)
 	} else {
 		password, err = readPassword(os.Stdin)
-		Verbosef("read password from stdin\n")
+		Verbosef("reading repository password from stdin\n")
 	}
 
 	if err != nil {
@@ -382,15 +391,41 @@ func ReadPasswordTwice(gopts GlobalOptions, prompt1, prompt2 string) (string, er
 	return pw1, nil
 }
 
+func ReadRepo(opts GlobalOptions) (string, error) {
+	if opts.Repo == "" && opts.RepositoryFile == "" {
+		return "", errors.Fatal("Please specify repository location (-r or --repository-file)")
+	}
+
+	repo := opts.Repo
+	if opts.RepositoryFile != "" {
+		if repo != "" {
+			return "", errors.Fatal("Options -r and --repository-file are mutually exclusive, please specify only one")
+		}
+
+		s, err := textfile.Read(opts.RepositoryFile)
+		if os.IsNotExist(errors.Cause(err)) {
+			return "", errors.Fatalf("%s does not exist", opts.RepositoryFile)
+		}
+		if err != nil {
+			return "", err
+		}
+
+		repo = strings.TrimSpace(string(s))
+	}
+
+	return repo, nil
+}
+
 const maxKeys = 20
 
 // OpenRepository reads the password and opens the repository.
 func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
-	if opts.Repo == "" {
-		return nil, errors.Fatal("Please specify repository location (-r)")
+	repo, err := ReadRepo(opts)
+	if err != nil {
+		return nil, err
 	}
 
-	be, err := open(opts.Repo, opts, opts.extended)
+	be, err := open(repo, opts, opts.extended)
 	if err != nil {
 		return nil, err
 	}
@@ -631,7 +666,7 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 
 // Open the backend specified by a location config.
 func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
-	debug.Log("parsing location %v", s)
+	debug.Log("parsing location %v", location.StripPassword(s))
 	loc, err := location.Parse(s)
 	if err != nil {
 		return nil, errors.Fatalf("parsing repository location failed: %v", err)
@@ -659,15 +694,11 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 
 	switch loc.Scheme {
 	case "local":
-		be, err = local.Open(cfg.(local.Config))
-		// wrap the backend in a LimitBackend so that the throughput is limited
-		be = limiter.LimitBackend(be, lim)
+		be, err = local.Open(globalOptions.ctx, cfg.(local.Config))
 	case "sftp":
-		be, err = sftp.Open(cfg.(sftp.Config))
-		// wrap the backend in a LimitBackend so that the throughput is limited
-		be = limiter.LimitBackend(be, lim)
+		be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config))
 	case "s3":
-		be, err = s3.Open(cfg.(s3.Config), rt)
+		be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt)
 	case "gs":
 		be, err = gs.Open(cfg.(gs.Config), rt)
 	case "azure":
@@ -686,13 +717,26 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	}
 
 	if err != nil {
-		return nil, errors.Fatalf("unable to open repo at %v: %v", s, err)
+		return nil, errors.Fatalf("unable to open repo at %v: %v", location.StripPassword(s), err)
+	}
+
+	// wrap backend if a test specified an inner hook
+	if gopts.backendInnerTestHook != nil {
+		be, err = gopts.backendInnerTestHook(be)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if loc.Scheme == "local" || loc.Scheme == "sftp" {
+		// wrap the backend in a LimitBackend so that the throughput is limited
+		be = limiter.LimitBackend(be, lim)
 	}
 
 	// check if config is there
 	fi, err := be.Stat(globalOptions.ctx, restic.Handle{Type: restic.ConfigFile})
 	if err != nil {
-		return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, s)
+		return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s))
 	}
 
 	if fi.Size == 0 {
@@ -726,11 +770,11 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 
 	switch loc.Scheme {
 	case "local":
-		return local.Create(cfg.(local.Config))
+		return local.Create(globalOptions.ctx, cfg.(local.Config))
 	case "sftp":
-		return sftp.Create(cfg.(sftp.Config))
+		return sftp.Create(globalOptions.ctx, cfg.(sftp.Config))
 	case "s3":
-		return s3.Create(cfg.(s3.Config), rt)
+		return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt)
 	case "gs":
 		return gs.Create(cfg.(gs.Config), rt)
 	case "azure":
@@ -740,9 +784,9 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "b2":
 		return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
 	case "rest":
-		return rest.Create(cfg.(rest.Config), rt)
+		return rest.Create(globalOptions.ctx, cfg.(rest.Config), rt)
 	case "rclone":
-		return rclone.Open(cfg.(rclone.Config), nil)
+		return rclone.Create(globalOptions.ctx, cfg.(rclone.Config))
 	}
 
 	debug.Log("invalid repository scheme: %v", s)

cmd/restic/global_test.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"bytes"
+	"io/ioutil"
+	"path/filepath"
 	"testing"
 
+	"github.com/restic/restic/internal/test"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -26,3 +29,33 @@ func Test_PrintFunctionsRespectsGlobalStdout(t *testing.T) {
 		buf.Reset()
 	}
 }
+
+func TestReadRepo(t *testing.T) {
+	tempDir, cleanup := test.TempDir(t)
+	defer cleanup()
+
+	// test --repo option
+	var opts GlobalOptions
+	opts.Repo = tempDir
+	repo, err := ReadRepo(opts)
+	rtest.OK(t, err)
+	rtest.Equals(t, tempDir, repo)
+
+	// test --repository-file option
+	foo := filepath.Join(tempDir, "foo")
+	err = ioutil.WriteFile(foo, []byte(tempDir+"\n"), 0666)
+	rtest.OK(t, err)
+
+	var opts2 GlobalOptions
+	opts2.RepositoryFile = foo
+	repo, err = ReadRepo(opts2)
+	rtest.OK(t, err)
+	rtest.Equals(t, tempDir, repo)
+
+	var opts3 GlobalOptions
+	opts3.RepositoryFile = foo + "-invalid"
+	_, err = ReadRepo(opts3)
+	if err == nil {
+		t.Fatal("must not read repository path from invalid file path")
+	}
+}

cmd/restic/integration_fuse_test.go

@@ -1,12 +1,8 @@
-// +build !netbsd
-// +build !openbsd
-// +build !solaris
-// +build !windows
+// +build darwin freebsd linux
 
 package main
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -122,7 +118,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 	}
 
 	for _, id := range snapshotIDs {
-		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
+		snapshot, err := restic.LoadSnapshot(global.ctx, repo, id)
 		rtest.OK(t, err)
 
 		ts := snapshot.Time.Format(time.RFC3339)
@@ -164,9 +160,6 @@ func TestMount(t *testing.T) {
 	repo, err := OpenRepository(env.gopts)
 	rtest.OK(t, err)
 
-	// We remove the mountpoint now to check that cmdMount creates it
-	rtest.RemoveAll(t, env.mountpoint)
-
 	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
 
 	rtest.SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))

cmd/restic/integration_helpers_windows_test.go

@@ -44,7 +44,6 @@ func createFileSetPerHardlink(dir string) map[uint64][]string {
 	}
 	for i, f := range files {
 		linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
-		i++
 	}
 	return linkTests
 }