Add version for 0.9.3

Output directory size in cache command

.github/ISSUE_TEMPLATE.md

@@ -1,63 +1,27 @@
 <!--
-NOTE: Not filling out the issue template needs a good reason, otherwise it may
-take a lot longer to find the problem! Please take the time to help us
-debugging the problem by collecting information, even if it seems irrelevant to
-you. Thanks!
 
-If you have a question, the forum at https://forum.restic.net is a better place.
-Please do not create issues for usage or documentation questions! We're using
-the GitHub issue tracker mainly for tracking bugs and feature requests.
--->
+Welcome! If you have a question or are unsure if you should open an issue,
+please use the forum instead!
 
-## Output of `restic version`
+   https://forum.restic.net
 
+The forum is a better place for questions about restic or general suggestions
+and topics, e.g. usage or documentation questions! This issue tracker is mainly
+for tracking bugs and feature requests directly relating to the development of
+the software itself, rather than the project.
 
-## How did you run restic exactly?
-
-<!--
-This section should include at least:
-
- * The complete command line and any environment variables you used to
-   configure restic's backend access. Make sure to replace sensitive values!
-
- * The output of the commands, what restic prints gives may give us much
-   information to diagnose the problem!
+Thanks for understanding, and for contributing to the project!
 -->
 
 
-## What backend/server/service did you use to store the repository?
-
-
-## Expected behavior
+Output of `restic version`
+--------------------------
 
 <!--
-Describe what you'd like restic to do differently.
+Please add the version of restic you're currently using here, this helps us
+later to see what has changed in restic when we revisit this issue after some
+time.
 -->
 
-## Actual behavior
-
-<!--
-In this section, please try to concentrate on observations, so only describe
-what you observed directly.
--->
-
-## Steps to reproduce the behavior
-
-<!--
-The more time you spend describing an easy way to reproduce the behavior (if
-this is possible), the easier it is for the project developers to fix it!
--->
-
-
-## Do you have any idea what may have caused this?
-
-
-## Do you have an idea how to solve the issue?
-
-## Did restic help you or made you happy in any way?
-
-<!--
-Answering this question is not required, but if you have anything positive to share, please do so here!
-Sometimes we get tired of reading bug reports all day and a little positive end note does wonders.
-Idea by Joey Hess, https://joeyh.name/blog/entry/two_holiday_stories/
--->
+Describe the issue
+------------------

.github/ISSUE_TEMPLATE/Bug.md

@@ -0,0 +1,93 @@
+---
+name: Bug report
+about: Report a problem with restic to help us resolve it and improve
+---
+
+<!--
+
+Welcome! - We kindly ask that you:
+
+  1. Fill out the issue template below - not doing so needs a good reason.
+  2. Use the forum if you have a question rather than a bug or feature request.
+
+The forum is at: https://forum.restic.net
+
+NOTE: Not filling out the issue template needs a good reason, as otherwise it
+may take a lot longer to find the problem, not to mention it can take up a lot
+more time which can otherwise be spent on development. Please also take the
+time to help us debug the issue by collecting relevant information, even if
+it doesn't seem to be relevant to you. Thanks!
+
+The forum is a better place for questions about restic or general suggestions
+and topics, e.g. usage or documentation questions! This issue tracker is mainly
+for tracking bugs and feature requests directly relating to the development of
+the software itself, rather than the project.
+
+Thanks for understanding, and for contributing to the project!
+
+-->
+
+
+Output of `restic version`
+--------------------------
+
+
+How did you run restic exactly?
+-------------------------------
+
+<!--
+This section should include at least:
+
+ * The complete command line and any environment variables you used to
+   configure restic's backend access. Make sure to replace sensitive values!
+
+ * The output of the commands, what restic prints gives may give us much
+   information to diagnose the problem!
+-->
+
+What backend/server/service did you use to store the repository?
+----------------------------------------------------------------
+
+
+
+Expected behavior
+-----------------
+
+<!--
+Describe what you'd like restic to do differently.
+-->
+
+Actual behavior
+---------------
+
+<!--
+In this section, please try to concentrate on observations, so only describe
+what you observed directly.
+-->
+
+Steps to reproduce the behavior
+-------------------------------
+
+<!--
+The more time you spend describing an easy way to reproduce the behavior (if
+this is possible), the easier it is for the project developers to fix it!
+-->
+
+Do you have any idea what may have caused this?
+-----------------------------------------------
+
+
+
+Do you have an idea how to solve the issue?
+-------------------------------------------
+
+
+
+Did restic help you or made you happy in any way?
+-------------------------------------------------
+
+<!--
+Answering this question is not required, but if you have anything positive to share, please do so here!
+Sometimes we get tired of reading bug reports all day and a little positive end note does wonders.
+Idea by Joey Hess, https://joeyh.name/blog/entry/two_holiday_stories/
+-->

.github/ISSUE_TEMPLATE/Feature.md

@@ -0,0 +1,57 @@
+---
+name: Feature request
+about: Suggest a new feature or enhancement for restic
+---
+
+<!--
+
+Welcome! - We kindly ask that you:
+
+  1. Fill out the issue template below - not doing so needs a good reason.
+  2. Use the forum if you have a question rather than a bug or feature request.
+
+The forum is at: https://forum.restic.net
+
+The forum is a better place for questions about restic or general suggestions
+and topics, e.g. usage or documentation questions! This issue tracker is mainly
+for tracking bugs and feature requests directly relating to the development of
+the software itself, rather than the project.
+
+Thanks for understanding, and for contributing to the project!
+
+-->
+
+
+Output of `restic version`
+--------------------------
+
+<!--
+Please add the version of restic you're currently using here, this helps us
+later to see what has changed in restic when we revisit this issue after some
+time.
+-->
+
+What should restic do differently? Which functionality do you think we should add?
+----------------------------------------------------------------------------------
+
+<!--
+Please describe the feature you'd like us to add here.
+-->
+
+
+What are you trying to do?
+--------------------------
+
+<!--
+This section should contain a brief description what you're trying to do, which
+would be possible after implementing the new feature.
+-->
+
+Did restic help you or made you happy in any way?
+-------------------------------------------------
+
+<!--
+Answering this question is not required, but if you have anything positive to share, please do so here!
+Sometimes we get tired of reading bug reports all day and a little positive end note does wonders.
+Idea by Joey Hess, https://joeyh.name/blog/entry/two_holiday_stories/
+-->

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,5 @@
+
+
 <!--
 Thank you very much for contributing code or documentation to restic! Please
 fill out the following questions to make it easier for us to review your
@@ -8,24 +10,27 @@ your time and add more commits. If you're done and ready for review, please
 check the last box.
 -->
 
-### What is the purpose of this change? What does it change?
+What is the purpose of this change? What does it change?
+--------------------------------------------------------
 
 <!--
 Describe the changes here, as detailed as needed.
 -->
 
-### Was the change discussed in an issue or in the forum before?
+Was the change discussed in an issue or in the forum before?
+------------------------------------------------------------
 
 <!--
 Link issues and relevant forum posts here.
 -->
 
-### Checklist
+Checklist
+---------
 
 - [ ] I have read the [Contribution Guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches)
 - [ ] I have added tests for all changes in this PR
 - [ ] I have added documentation for the changes (in the manual)
-- [ ] There's a new file in a subdir of `changelog/x.y.z` that describe the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/changelog-entry.tmpl))
+- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/TEMPLATE))
 - [ ] I have run `gofmt` on the code in all commits
 - [ ] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits)
 - [ ] I'm done, this Pull Request is ready for review

.gitignore

@@ -1,3 +1,2 @@
 /restic
 /.vagrant
-/doc/_build

.travis.yml

@@ -1,33 +1,40 @@
 language: go
 sudo: false
 
-go:
-  - "1.8.x"
-  - "1.9.x"
-  - "1.10"
-
-os:
-  - linux
-  - osx
-
-env:
-  matrix:
-    RESTIC_TEST_FUSE=0
-
 matrix:
-  exclude:
-    - os: osx
-      go: "1.8.x"
-    - os: osx
-      go: "1.9.x"
-    - os: linux
-      go: "1.10"
   include:
     - os: linux
-      go: "1.10"
+      go: "1.9.x"
+      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0 RESTIC_BUILD_SOLARIS=0
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
+
+    - os: linux
+      go: "1.10.x"
+      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
+
+    # only run fuse and cloud backends tests on Travis for the latest Go on Linux
+    - os: linux
+      go: "1.11.x"
       sudo: true
-      env:
-        RESTIC_TEST_FUSE=1
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
+
+    - os: osx
+      go: "1.11.x"
+      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/Library/Caches/go-build
+          - $HOME/gopath/pkg/mod
 
 branches:
   only:
@@ -51,4 +58,4 @@ script:
   - go run run_integration_tests.go
 
 after_success:
-  - bash <(curl -s https://codecov.io/bash) -f all.cov
+  - test -r all.cov && bash <(curl -s https://codecov.io/bash) -f all.cov

CHANGELOG.md

@@ -1,3 +1,690 @@
+Changelog for restic 0.9.3 (2018-10-13)
+=======================================
+
+The following sections list the changes in restic 0.9.3 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1935: Remove truncated files from cache
+ * Fix #1978: Do not return an error when the scanner is faster than backup
+ * Enh #1766: Restore: suppress lchown errors when not running as root
+ * Enh #1909: Reject files/dirs by name first
+ * Enh #1940: Add directory filter to ls command
+ * Enh #1967: Use `--host` everywhere
+ * Enh #2028: Display size of cache directories
+ * Enh #1777: Improve the `find` command
+ * Enh #1876: Display reason why forget keeps snapshots
+ * Enh #1891: Accept glob in paths loaded via --files-from
+ * Enh #1920: Vendor dependencies with Go 1.11 Modules
+ * Enh #1949: Add new command `self-update`
+ * Enh #1953: Ls: Add JSON output support for restic ls cmd
+
+Details
+-------
+
+ * Bugfix #1935: Remove truncated files from cache
+
+   When a file in the local cache is truncated, and restic tries to access data beyond the end of the
+   (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are
+   removed and the data is fetched directly from the backend.
+
+   https://github.com/restic/restic/issues/1935
+
+ * Bugfix #1978: Do not return an error when the scanner is faster than backup
+
+   When restic makes a backup, there's a background task called "scanner" which collects
+   information on how many files and directories are to be saved, in order to display progress
+   information to the user. When the backup finishes faster than the scanner, it is aborted
+   because the result is not needed any more. This logic contained a bug, where quitting the
+   scanner process was treated as an error, and caused restic to print an unhelpful error message
+   ("context canceled").
+
+   https://github.com/restic/restic/issues/1978
+   https://github.com/restic/restic/pull/1991
+
+ * Enhancement #1766: Restore: suppress lchown errors when not running as root
+
+   Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files
+   during restore if it is run as root, on non-Windows operating systems. On Windows, the error
+   is reported as usual.
+
+   https://github.com/restic/restic/issues/1766
+
+ * Enhancement #1909: Reject files/dirs by name first
+
+   The current scanner/archiver code had an architectural limitation: it always ran the
+   `lstat()` system call on all files and directories before a decision to include/exclude the
+   file/dir was made. This lead to a lot of unnecessary system calls for items that could have been
+   rejected by their name or path only.
+
+   We've changed the archiver/scanner implementation so that it now first rejects by name/path,
+   and only runs the system call on the remaining items. This reduces the number of `lstat()`
+   system calls a lot (depending on the exclude settings).
+
+   https://github.com/restic/restic/issues/1909
+   https://github.com/restic/restic/pull/1912
+
+ * Enhancement #1940: Add directory filter to ls command
+
+   The ls command can now be filtered by directories, so that only files in the given directories
+   will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list
+   their files as well.
+
+   It used to be possible to specify multiple snapshots, but that has been replaced by only one
+   snapshot and the possibility of specifying multiple directories.
+
+   Specifying directories constrains the walk, which can significantly speed up the listing.
+
+   https://github.com/restic/restic/issues/1940
+   https://github.com/restic/restic/pull/1941
+
+ * Enhancement #1967: Use `--host` everywhere
+
+   We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g.
+   for `restic backup`) still works, but will print a deprecation warning. Also, add the short
+   option `-H` where possible.
+
+   https://github.com/restic/restic/issues/1967
+
+ * Enhancement #2028: Display size of cache directories
+
+   The `cache` command now by default shows the size of the individual cache directories. It can be
+   disabled with `--no-size`.
+
+   https://github.com/restic/restic/issues/2028
+   https://github.com/restic/restic/pull/2033
+
+ * Enhancement #1777: Improve the `find` command
+
+   We've updated the `find` command to support multiple patterns.
+
+   `restic find` is now able to list the snapshots containing a specific tree or blob, or even the
+   snapshots that contain blobs belonging to a given pack. A list of IDs can be given, as long as they
+   all have the same type.
+
+   The command `find` can also display the pack IDs the blobs belong to, if the `--show-pack-id`
+   flag is provided.
+
+   https://github.com/restic/restic/issues/1777
+   https://github.com/restic/restic/pull/1780
+
+ * Enhancement #1876: Display reason why forget keeps snapshots
+
+   We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a
+   particuliar snapshot. This makes debugging policies for forget much easier. Please remember
+   to always try things out with `--dry-run`!
+
+   https://github.com/restic/restic/pull/1876
+
+ * Enhancement #1891: Accept glob in paths loaded via --files-from
+
+   Before that, behaviour was different if paths were appended to command line or from a file,
+   because wild card characters were expanded by shell if appended to command line, but not
+   expanded if loaded from file.
+
+   https://github.com/restic/restic/issues/1891
+
+ * Enhancement #1920: Vendor dependencies with Go 1.11 Modules
+
+   Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules.
+   For users this does not change much, only if you want to compile restic without downloading
+   anything with Go 1.11, then you need to run: `go build -mod=vendor build.go`
+
+   https://github.com/restic/restic/pull/1920
+
+ * Enhancement #1949: Add new command `self-update`
+
+   We have added a new command called `self-update` which downloads the latest released version
+   of restic from GitHub and replaces the current binary with it. It does not rely on any external
+   program (so it'll work everywhere), but still verifies the GPG signature using the embedded
+   GPG public key.
+
+   By default, the `self-update` command is hidden behind the `selfupdate` built tag, which is
+   only set when restic is built using `build.go` (including official releases). The reason for
+   this is that downstream distributions will then not include the command by default, so users
+   are encouraged to use the platform-specific distribution mechanism.
+
+   https://github.com/restic/restic/pull/1949
+
+ * Enhancement #1953: Ls: Add JSON output support for restic ls cmd
+
+   We've implemented listing files in the repository with JSON as output, just pass `--json` as an
+   option to `restic ls`. This makes the output of the command machine readable.
+
+   https://github.com/restic/restic/pull/1953
+
+
+Changelog for restic 0.9.2 (2018-08-06)
+=======================================
+
+The following sections list the changes in restic 0.9.2 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1854: Allow saving files/dirs on different fs with `--one-file-system`
+ * Fix #1870: Fix restore with --include
+ * Fix #1880: Use `--cache-dir` argument for `check` command
+ * Fix #1893: Return error when exclude file cannot be read
+ * Fix #1861: Fix case-insensitive search with restic find
+ * Enh #1906: Add support for B2 application keys
+ * Enh #874: Add stats command to get information about a repository
+ * Enh #1772: Add restore --verify to verify restored file content
+ * Enh #1853: Add JSON output support to `restic key list`
+ * Enh #1477: S3 backend: accept AWS_SESSION_TOKEN
+ * Enh #1901: Update the Backblaze B2 library
+
+Details
+-------
+
+ * Bugfix #1854: Allow saving files/dirs on different fs with `--one-file-system`
+
+   Restic now allows saving files/dirs on a different file system in a subdir correctly even when
+   `--one-file-system` is specified.
+
+   The first thing the restic archiver code does is to build a tree of the target
+   files/directories. If it detects that a parent directory is already included (e.g. `restic
+   backup /foo /foo/bar/baz`), it'll ignore the latter argument.
+
+   Without `--one-file-system`, that's perfectly valid: If `/foo` is to be archived, it will
+   include `/foo/bar/baz`. But with `--one-file-system`, `/foo/bar/baz` may reside on a
+   different file system, so it won't be included with `/foo`.
+
+   https://github.com/restic/restic/issues/1854
+   https://github.com/restic/restic/pull/1855
+
+ * Bugfix #1870: Fix restore with --include
+
+   We fixed a bug which prevented restic to restore files with an include filter.
+
+   https://github.com/restic/restic/issues/1870
+   https://github.com/restic/restic/pull/1900
+
+ * Bugfix #1880: Use `--cache-dir` argument for `check` command
+
+   `check` command now uses a temporary sub-directory of the specified directory if set using the
+   `--cache-dir` argument. If not set, the cache directory is created in the default temporary
+   directory as before. In either case a temporary cache is used to ensure the actual repository is
+   checked (rather than a local copy).
+
+   The `--cache-dir` argument was not used by the `check` command, instead a cache directory was
+   created in the temporary directory.
+
+   https://github.com/restic/restic/issues/1880
+
+ * Bugfix #1893: Return error when exclude file cannot be read
+
+   A bug was found: when multiple exclude files were passed to restic and one of them could not be
+   read, an error was printed and restic continued, ignoring even the existing exclude files.
+   Now, an error message is printed and restic aborts when an exclude file cannot be read.
+
+   https://github.com/restic/restic/issues/1893
+
+ * Bugfix #1861: Fix case-insensitive search with restic find
+
+   We've fixed the behavior for `restic find -i PATTERN`, which was broken in v0.9.1.
+
+   https://github.com/restic/restic/pull/1861
+
+ * Enhancement #1906: Add support for B2 application keys
+
+   Restic can now use so-called "application keys" which can be created in the B2 dashboard and
+   were only introduced recently. In contrast to the "master key", such keys can be restricted to a
+   specific bucket and/or path.
+
+   https://github.com/restic/restic/issues/1906
+   https://github.com/restic/restic/pull/1914
+
+ * Enhancement #874: Add stats command to get information about a repository
+
+   https://github.com/restic/restic/issues/874
+   https://github.com/restic/restic/pull/1729
+
+ * Enhancement #1772: Add restore --verify to verify restored file content
+
+   Restore will print error message if restored file content does not match expected SHA256
+   checksum
+
+   https://github.com/restic/restic/pull/1772
+
+ * Enhancement #1853: Add JSON output support to `restic key list`
+
+   This PR enables users to get the output of `restic key list` in JSON in addition to the existing
+   table format.
+
+   https://github.com/restic/restic/pull/1853
+
+ * Enhancement #1477: S3 backend: accept AWS_SESSION_TOKEN
+
+   Before, it was not possible to use s3 backend with AWS temporary security credentials(with
+   AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials
+   provider.
+
+   https://github.com/restic/restic/issues/1477
+   https://github.com/restic/restic/pull/1479
+   https://github.com/restic/restic/pull/1647
+
+ * Enhancement #1901: Update the Backblaze B2 library
+
+   We've updated the library we're using for accessing the Backblaze B2 service to 0.5.0 to
+   include support for upcoming so-called "application keys". With this feature, you can create
+   access credentials for B2 which are restricted to e.g. a single bucket or even a sub-directory
+   of a bucket.
+
+   https://github.com/restic/restic/pull/1901
+   https://github.com/kurin/blazer
+
+
+Changelog for restic 0.9.1 (2018-06-10)
+=======================================
+
+The following sections list the changes in restic 0.9.1 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1801: Add limiting bandwidth to the rclone backend
+ * Fix #1822: Allow uploading large files to MS Azure
+ * Fix #1825: Correct `find` to not skip snapshots
+ * Fix #1833: Fix caching files on error
+ * Fix #1834: Resolve deadlock
+
+Details
+-------
+
+ * Bugfix #1801: Add limiting bandwidth to the rclone backend
+
+   The rclone backend did not respect `--limit-upload` or `--limit-download`. Oftentimes it's
+   not necessary to use this, as the limiting in rclone itself should be used because it gives much
+   better results, but in case a remote instance of rclone is used (e.g. called via ssh), it is still
+   relevant to limit the bandwidth from restic to rclone.
+
+   https://github.com/restic/restic/issues/1801
+
+ * Bugfix #1822: Allow uploading large files to MS Azure
+
+   Sometimes, restic creates files to be uploaded to the repository which are quite large, e.g.
+   when saving directories with many entries or very large files. The MS Azure API does not allow
+   uploading files larger that 256MiB directly, rather restic needs to upload them in blocks of
+   100MiB. This is now implemented.
+
+   https://github.com/restic/restic/issues/1822
+
+ * Bugfix #1825: Correct `find` to not skip snapshots
+
+   Under certain circumstances, the `find` command was found to skip snapshots containing
+   directories with files to look for when the directories haven't been modified at all, and were
+   already printed as part of a different snapshot. This is now corrected.
+
+   In addition, we've switched to our own matching/pattern implementation, so now things like
+   `restic find "/home/user/foo/**/main.go"` are possible.
+
+   https://github.com/restic/restic/issues/1825
+   https://github.com/restic/restic/issues/1823
+
+ * Bugfix #1833: Fix caching files on error
+
+   During `check` it may happen that different threads access the same file in the backend, which
+   is then downloaded into the cache only once. When that fails, only the thread which is
+   responsible for downloading the file signals the correct error. The other threads just assume
+   that the file has been downloaded successfully and then get an error when they try to access the
+   cached file.
+
+   https://github.com/restic/restic/issues/1833
+
+ * Bugfix #1834: Resolve deadlock
+
+   When the "scanning" process restic runs to find out how much data there is does not finish before
+   the backup itself is done, restic stops doing anything. This is resolved now.
+
+   https://github.com/restic/restic/issues/1834
+   https://github.com/restic/restic/pull/1835
+
+
+Changelog for restic 0.9.0 (2018-05-21)
+=======================================
+
+The following sections list the changes in restic 0.9.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1608: Respect time stamp for new backup when reading from stdin
+ * Fix #1652: Ignore/remove invalid lock files
+ * Fix #1730: Ignore sockets for restore
+ * Fix #1684: Fix backend tests for rest-server
+ * Fix #1745: Correctly parse the argument to --tls-client-cert
+ * Enh #1433: Support UTF-16 encoding and process Byte Order Mark
+ * Enh #1561: Allow using rclone to access other services
+ * Enh #1665: Improve cache handling for `restic check`
+ * Enh #1721: Add `cache` command to list cache dirs
+ * Enh #1758: Allow saving OneDrive folders in Windows
+ * Enh #549: Rework archiver code
+ * Enh #1552: Use Google Application Default credentials
+ * Enh #1477: Accept AWS_SESSION_TOKEN for the s3 backend
+ * Enh #1648: Ignore AWS permission denied error when creating a repository
+ * Enh #1649: Add illumos/Solaris support
+ * Enh #1709: Improve messages `restic check` prints
+ * Enh #827: Add --new-password-file flag for non-interactive password changes
+ * Enh #1735: Allow keeping a time range of snaphots
+ * Enh #1782: Use default AWS credentials chain for S3 backend
+
+Details
+-------
+
+ * Bugfix #1608: Respect time stamp for new backup when reading from stdin
+
+   When reading backups from stdin (via `restic backup --stdin`), restic now uses the time stamp
+   for the new backup passed in `--time`.
+
+   https://github.com/restic/restic/issues/1608
+   https://github.com/restic/restic/pull/1703
+
+ * Bugfix #1652: Ignore/remove invalid lock files
+
+   This corrects a bug introduced recently: When an invalid lock file in the repo is encountered
+   (e.g. if the file is empty), the code used to ignore that, but now returns the error. Now, invalid
+   files are ignored for the normal lock check, and removed when `restic unlock --remove-all` is
+   run.
+
+   https://github.com/restic/restic/issues/1652
+   https://github.com/restic/restic/pull/1653
+
+ * Bugfix #1730: Ignore sockets for restore
+
+   We've received a report and correct the behavior in which the restore code aborted restoring a
+   directory when a socket was encountered. Unix domain socket files cannot be restored (they are
+   created on the fly once a process starts listening). The error handling was corrected, and in
+   addition we're now ignoring sockets during restore.
+
+   https://github.com/restic/restic/issues/1730
+   https://github.com/restic/restic/pull/1731
+
+ * Bugfix #1684: Fix backend tests for rest-server
+
+   The REST server for restic now requires an explicit parameter (`--no-auth`) if no
+   authentication should be allowed. This is fixed in the tests.
+
+   https://github.com/restic/restic/pull/1684
+
+ * Bugfix #1745: Correctly parse the argument to --tls-client-cert
+
+   Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) instead of the
+   argument that was passed to it. This has been corrected.
+
+   https://github.com/restic/restic/issues/1745
+   https://github.com/restic/restic/pull/1746
+
+ * Enhancement #1433: Support UTF-16 encoding and process Byte Order Mark
+
+   On Windows, text editors commonly leave a Byte Order Mark at the beginning of the file to define
+   which encoding is used (oftentimes UTF-16). We've added code to support processing the BOMs in
+   text files, like the exclude files, the password file and the file passed via `--files-from`.
+   This does not apply to any file being saved in a backup, those are not touched and archived as they
+   are.
+
+   https://github.com/restic/restic/issues/1433
+   https://github.com/restic/restic/issues/1738
+   https://github.com/restic/restic/pull/1748
+
+ * Enhancement #1561: Allow using rclone to access other services
+
+   We've added the ability to use rclone to store backup data on all backends that it supports. This
+   was done in collaboration with Nick, the author of rclone. You can now use it to first configure a
+   service, then restic manages the rest (starting and stopping rclone). For details, please see
+   the manual.
+
+   https://github.com/restic/restic/issues/1561
+   https://github.com/restic/restic/pull/1657
+   https://rclone.org
+
+ * Enhancement #1665: Improve cache handling for `restic check`
+
+   For safety reasons, restic does not use a local metadata cache for the `restic check` command,
+   so that data is loaded from the repository and restic can check it's in good condition. When the
+   cache is disabled, restic will fetch each tiny blob needed for checking the integrity using a
+   separate backend request. For non-local backends, that will take a long time, and depending on
+   the backend (e.g. B2) may also be much more expensive.
+
+   This PR adds a few commits which will change the behavior as follows:
+
+   * When `restic check` is called without any additional parameters, it will build a new cache in a
+   temporary directory, which is removed at the end of the check. This way, we'll get readahead for
+   metadata files (so restic will fetch the whole file when the first blob from the file is
+   requested), but all data is freshly fetched from the storage backend. This is the default
+   behavior and will work for almost all users.
+
+   * When `restic check` is called with `--with-cache`, the default on-disc cache is used. This
+   behavior hasn't changed since the cache was introduced.
+
+   * When `--no-cache` is specified, restic falls back to the old behavior, and read all tiny blobs
+   in separate requests.
+
+   https://github.com/restic/restic/issues/1665
+   https://github.com/restic/restic/issues/1694
+   https://github.com/restic/restic/pull/1696
+
+ * Enhancement #1721: Add `cache` command to list cache dirs
+
+   The command `cache` was added, it allows listing restic's cache directoriers together with
+   the last usage. It also allows removing old cache dirs without having to access a repo, via
+   `restic cache --cleanup`
+
+   https://github.com/restic/restic/issues/1721
+   https://github.com/restic/restic/pull/1749
+
+ * Enhancement #1758: Allow saving OneDrive folders in Windows
+
+   Restic now contains a bugfix to two libraries, which allows saving OneDrive folders in
+   Windows. In order to use the newer versions of the libraries, the minimal version required to
+   compile restic is now Go 1.9.
+
+   https://github.com/restic/restic/issues/1758
+   https://github.com/restic/restic/pull/1765
+
+ * Enhancement #549: Rework archiver code
+
+   The core archiver code and the complementary code for the `backup` command was rewritten
+   completely. This resolves very annoying issues such as 549. The first backup with this release
+   of restic will likely result in all files being re-read locally, so it will take a lot longer. The
+   next backup after that will be fast again.
+
+   Basically, with the old code, restic took the last path component of each to-be-saved file or
+   directory as the top-level file/directory within the snapshot. This meant that when called as
+   `restic backup /home/user/foo`, the snapshot would contain the files in the directory
+   `/home/user/foo` as `/foo`.
+
+   This is not the case any more with the new archiver code. Now, restic works very similar to what
+   `tar` does: When restic is called with an absolute path to save, then it'll preserve the
+   directory structure within the snapshot. For the example above, the snapshot would contain
+   the files in the directory within `/home/user/foo` in the snapshot. For relative
+   directories, it only preserves the relative path components. So `restic backup user/foo`
+   will save the files as `/user/foo` in the snapshot.
+
+   While we were at it, the status display and notification system was completely rewritten. By
+   default, restic now shows which files are currently read (unless `--quiet` is specified) in a
+   multi-line status display.
+
+   The `backup` command also gained a new option: `--verbose`. It can be specified once (which
+   prints a bit more detail what restic is doing) or twice (which prints a line for each
+   file/directory restic encountered, together with some statistics).
+
+   Another issue that was resolved is the new code only reads two files at most. The old code would
+   read way too many files in parallel, thereby slowing down the backup process on spinning discs a
+   lot.
+
+   https://github.com/restic/restic/issues/549
+   https://github.com/restic/restic/issues/1286
+   https://github.com/restic/restic/issues/446
+   https://github.com/restic/restic/issues/1344
+   https://github.com/restic/restic/issues/1416
+   https://github.com/restic/restic/issues/1456
+   https://github.com/restic/restic/issues/1145
+   https://github.com/restic/restic/issues/1160
+   https://github.com/restic/restic/pull/1494
+
+ * Enhancement #1552: Use Google Application Default credentials
+
+   Google provide libraries to generate appropriate credentials with various fallback
+   sources. This change uses the library to generate our GCS client, which allows us to make use of
+   these extra methods.
+
+   This should be backward compatible with previous restic behaviour while adding the
+   additional capabilities to auth from Google's internal metadata endpoints. For users
+   running restic in GCP this can make authentication far easier than it was before.
+
+   https://github.com/restic/restic/pull/1552
+   https://developers.google.com/identity/protocols/application-default-credentials
+
+ * Enhancement #1477: Accept AWS_SESSION_TOKEN for the s3 backend
+
+   Before, it was not possible to use s3 backend with AWS temporary security credentials(with
+   AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials
+   provider.
+
+   https://github.com/restic/restic/issues/1477
+   https://github.com/restic/restic/pull/1479
+   https://github.com/restic/restic/pull/1647
+
+ * Enhancement #1648: Ignore AWS permission denied error when creating a repository
+
+   It's not possible to use s3 backend scoped to a subdirectory(with specific permissions).
+   Restic doesn't try to create repository in a subdirectory, when 'bucket exists' of parent
+   directory check fails due to permission issues.
+
+   https://github.com/restic/restic/pull/1648
+
+ * Enhancement #1649: Add illumos/Solaris support
+
+   https://github.com/restic/restic/pull/1649
+
+ * Enhancement #1709: Improve messages `restic check` prints
+
+   Some messages `restic check` prints are not really errors, so from now on restic does not treat
+   them as errors any more and exits cleanly.
+
+   https://github.com/restic/restic/pull/1709
+   https://forum.restic.net/t/what-is-the-standard-procedure-to-follow-if-a-backup-or-restore-is-interrupted/571/2
+
+ * Enhancement #827: Add --new-password-file flag for non-interactive password changes
+
+   This makes it possible to change a repository password without being prompted.
+
+   https://github.com/restic/restic/issues/827
+   https://github.com/restic/restic/pull/1720
+   https://forum.restic.net/t/changing-repo-password-without-prompt/591
+
+ * Enhancement #1735: Allow keeping a time range of snaphots
+
+   We've added the `--keep-within` option to the `forget` command. It instructs restic to keep
+   all snapshots within the given duration since the newest snapshot. For example, running
+   `restic forget --keep-within 5m7d` will keep all snapshots which have been made in the five
+   months and seven days since the latest snapshot.
+
+   https://github.com/restic/restic/pull/1735
+
+ * Enhancement #1782: Use default AWS credentials chain for S3 backend
+
+   Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), and reorders
+   the credentials chain for the S3 backend to match AWS's standard, which is static credentials,
+   env vars, credentials file, and finally remote.
+
+   https://github.com/restic/restic/pull/1782
+
+
+Changelog for restic 0.8.3 (2018-02-26)
+=======================================
+
+The following sections list the changes in restic 0.8.3 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1633: Fixed unexpected 'pack file cannot be listed' error
+ * Fix #1641: Ignore files with invalid names in the repo
+ * Fix #1638: Handle errors listing files in the backend
+ * Enh #1497: Add --read-data-subset flag to check command
+ * Enh #1560: Retry all repository file download errors
+ * Enh #1623: Don't check for presence of files in the backend before writing
+ * Enh #1634: Upgrade B2 client library, reduce HTTP requests
+
+Details
+-------
+
+ * Bugfix #1633: Fixed unexpected 'pack file cannot be listed' error
+
+   Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune` commands failed to
+   read pack files with size of 587, 588, 589 or 590 bytes.
+
+   https://github.com/restic/restic/issues/1633
+   https://github.com/restic/restic/pull/1635
+
+ * Bugfix #1641: Ignore files with invalid names in the repo
+
+   The release 0.8.2 introduced a bug: when restic encounters files in the repo which do not have a
+   valid name, it tries to load a file with a name of lots of zeroes instead of ignoring it. This is now
+   resolved, invalid file names are just ignored.
+
+   https://github.com/restic/restic/issues/1641
+   https://github.com/restic/restic/pull/1643
+   https://forum.restic.net/t/help-fixing-repo-no-such-file/485/3
+
+ * Bugfix #1638: Handle errors listing files in the backend
+
+   A user reported in the forum that restic completes a backup although a concurrent `prune`
+   operation was running. A few error messages were printed, but the backup was attempted and
+   completed successfully. No error code was returned.
+
+   This should not happen: The repository is exclusively locked during `prune`, so when `restic
+   backup` is run in parallel, it should abort and return an error code instead.
+
+   It was found that the bug was in the code introduced only recently, which retries a List()
+   operation on the backend should that fail. It is now corrected.
+
+   https://github.com/restic/restic/pull/1638
+   https://forum.restic.net/t/restic-backup-returns-0-exit-code-when-already-locked/484
+
+ * Enhancement #1497: Add --read-data-subset flag to check command
+
+   This change introduces ability to check integrity of a subset of repository data packs. This
+   can be used to spread integrity check of larger repositories over a period of time.
+
+   https://github.com/restic/restic/issues/1497
+   https://github.com/restic/restic/pull/1556
+
+ * Enhancement #1560: Retry all repository file download errors
+
+   Restic will now retry failed downloads, similar to other operations.
+
+   https://github.com/restic/restic/pull/1560
+
+ * Enhancement #1623: Don't check for presence of files in the backend before writing
+
+   Before, all backend implementations were required to return an error if the file that is to be
+   written already exists in the backend. For most backends, that means making a request (e.g. via
+   HTTP) and returning an error when the file already exists.
+
+   This is not accurate, the file could have been created between the HTTP request testing for it,
+   and when writing starts, so we've relaxed this requeriment, which saves one additional HTTP
+   request per newly added file.
+
+   https://github.com/restic/restic/pull/1623
+
+ * Enhancement #1634: Upgrade B2 client library, reduce HTTP requests
+
+   We've upgraded the B2 client library restic uses to access BackBlaze B2. This reduces the
+   number of HTTP requests needed to upload a new file from two to one, which should improve
+   throughput to B2.
+
+   https://github.com/restic/restic/pull/1634
+
+
 Changelog for restic 0.8.2 (2018-02-17)
 =======================================
 
@@ -69,6 +756,7 @@ Details
    of data loss, just minor inconvenience for our users.
 
    https://github.com/restic/restic/pull/1589
+   https://forum.restic.net/t/error-loading-tree-check-prune-and-forget-gives-error-b2-backend/406
 
  * Bugfix #1594: Google Cloud Storage: Use generic HTTP transport
 
@@ -607,7 +1295,7 @@ Details
  * Enhancement #1203: Print stats on all BSD systems when SIGINFO (ctrl+t) is received
 
    https://github.com/restic/restic/pull/1203
-   https://github.com/restic/restic/pull/1082
+   https://github.com/restic/restic/pull/1082#issuecomment-326279920
 
  * Enhancement #1205: Allow specifying time/date for a backup with `--time`
 
@@ -647,12 +1335,12 @@ Details
  * Enhancement #1055: Create subdirs below `data/` for local/sftp backends
 
    The local and sftp backends now create the subdirs below `data/` on open/init. This way, restic
-   makes sure that they always exist. This is connected to an issue for the sftp server:
+   makes sure that they always exist. This is connected to an issue for the sftp server.
 
    https://github.com/restic/restic/issues/1055
+   https://github.com/restic/rest-server/pull/11#issuecomment-309879710
    https://github.com/restic/restic/pull/1077
    https://github.com/restic/restic/pull/1105
-   https://github.com/restic/rest-server/pull/11#issuecomment-309879710
 
  * Enhancement #1067: Allow loading credentials for s3 from IAM
 
@@ -664,7 +1352,7 @@ Details
 
  * Enhancement #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
 
-   The `migrate` command for chaning the `s3legacy` layout to the `default` layout for s3
+   The `migrate` command for changing the `s3legacy` layout to the `default` layout for s3
    backends has been improved: It can now be restarted with `restic migrate --force s3_layout`
    and automatically retries operations on error.
 

CONTRIBUTING.md

@@ -46,12 +46,15 @@ Remember, the easier it is for us to reproduce the bug, the earlier it will be
 corrected!
 
 In addition, you can compile restic with debug support by running
-`go run build.go -tags debug` and instructing it to create a debug log by
-setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
+`go run -mod=vendor build.go -tags debug` and instructing it to create a debug
+log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
 
     $ export DEBUG_LOG=/tmp/restic-debug.log
     $ restic backup ~/work
 
+For Go < 1.11, you need to remove the `-mod=vendor` option from the build
+command.
+
 Please be aware that the debug log file will contain potentially sensitive
 things like file and directory names, so please either redact it before
 uploading it somewhere or post only the parts that are really relevant.
@@ -60,9 +63,37 @@ uploading it somewhere or post only the parts that are really relevant.
 Development Environment
 =======================
 
-In order to compile restic with the `go` tool directly, it needs to be checked
-out at the right path within a `GOPATH`. The concept of a `GOPATH` is explained
-in ["How to write Go code"](https://golang.org/doc/code.html).
+The repository contains several sets of directories with code: `cmd/` and
+`internal/` contain the code written for restic, whereas `vendor/` contains
+copies of libraries restic depends on. The libraries are managed with the
+command `go mod vendor`.
+
+Go >= 1.11
+----------
+
+For Go version 1.11 or later, you should clone the repo (without having
+`$GOPATH` set) and `cd` into the directory:
+
+    $ unset GOPATH
+    $ git clone https://github.com/restic/restic
+    $ cd restic
+
+Then use the `go` tool to build restic:
+
+    $ go build ./cmd/restic
+    $ ./restic version
+    restic 0.9.2-dev (compiled manually) compiled with go1.11 on linux/amd64
+
+You can run all tests with the following command:
+
+    $ go test ./...
+
+Go < 1.11
+---------
+
+In order to compile restic with Go before 1.11, it needs to be checked out at
+the right path within a `GOPATH`. The concept of a `GOPATH` is explained in
+["How to write Go code"](https://golang.org/doc/code.html).
 
 If you do not have a directory with Go code yet, executing the following
 instructions in your shell will create one for you and check out the restic
@@ -83,12 +114,7 @@ You can then build restic as follows:
 
 The following commands can be used to run all the tests:
 
-    $ go test ./cmd/... ./internal/...
-
-The repository contains two sets of directories with code: `cmd/` and
-`internal/` contain the code written for restic, whereas `vendor/` contains
-copies of libraries restic depends on. The libraries are managed with the
-[`dep`](https://github.com/golang/dep) tool.
+    $ go test ./...
 
 Providing Patches
 =================
@@ -141,13 +167,14 @@ run
 
     gofmt -w **/*.go
 
-in the project root directory before committing. Installing the script
-`fmt-check` from https://github.com/edsrzf/gofmt-git-hook locally as a
-pre-commit hook checks formatting before committing automatically, just copy
-this script to `.git/hooks/pre-commit`.
+in the project root directory before committing. For each Pull Request, the
+formatting is tested with `gofmt` for the latest stable version of Go.
+Installing the script `fmt-check` from https://github.com/edsrzf/gofmt-git-hook
+locally as a pre-commit hook checks formatting before committing automatically,
+just copy this script to `.git/hooks/pre-commit`.
 
 For each pull request, several different systems run the integration tests on
-Linux, OS X and Windows. We won't merge any code that does not pass all tests
+Linux, macOS and Windows. We won't merge any code that does not pass all tests
 for all systems, so when a tests fails, try to find out what's wrong and fix
 it. If you need help on this, please leave a comment in the pull request, and
 we'll be glad to assist. Having a PR with failing integration tests is nothing
@@ -164,7 +191,7 @@ history and triaging bugs much easier.
 Git commit messages have a very terse summary in the first line of the commit
 message, followed by an empty line, followed by a more verbose description or a
 List of changed things. For examples, please refer to the excellent [How to
-Write a Git Commit Message](http://chris.beams.io/posts/git-commit/).
+Write a Git Commit Message](https://chris.beams.io/posts/git-commit/).
 
 If you change/add multiple different things that aren't related at all, try to
 make several smaller commits. This is much easier to review. Using `git add -p`

GOVERNANCE.md

@@ -0,0 +1,27 @@
+# restic project governance
+
+## Overview
+
+The restic project uses a governance model commonly described as Benevolent
+Dictator For Life (BDFL). This document outlines our understanding of what this
+means. It is derived from the [i3 window manager project
+governance](https://raw.githubusercontent.com/i3/i3/next/.github/GOVERNANCE.md).
+
+## Roles
+
+* user: anyone who interacts with the restic project
+* core contributor: a handful of people who have contributed significantly to
+  the project by any means (issue triage, support, documentation, code, etc.).
+  Core contributors are recognizable via GitHub’s "Member" badge.
+* Benevolent Dictator For Life (BDFL): a single individual who makes decisions
+  when consensus cannot be reached. restic's current BDFL is [@fd0](https://github.com/fd0).
+
+## Decision making process
+
+In general, we try to reach consensus in discussions. In case consensus cannot
+be reached, the BDFL makes a decision.
+
+## Contribution process
+
+The contribution process is described in a separate document called
+[CONTRIBUTING](CONTRIBUTING.md).

Gopkg.lock

@@ -1,236 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  name = "bazil.org/fuse"
-  packages = [".","fs","fuseutil"]
-  revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748"
-
-[[projects]]
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  revision = "767c40d6a2e058483c25fa193e963a22da17236d"
-  version = "v0.18.0"
-
-[[projects]]
-  name = "github.com/Azure/azure-sdk-for-go"
-  packages = ["storage"]
-  revision = "eae258195456be76b2ec9ad2ee2ab63cdda365d9"
-  version = "v12.2.0-beta"
-
-[[projects]]
-  name = "github.com/Azure/go-autorest"
-  packages = ["autorest","autorest/adal","autorest/azure","autorest/date"]
-  revision = "c2a68353555b68de3ee8455a4fd3e890a0ac6d99"
-  version = "v9.8.1"
-
-[[projects]]
-  name = "github.com/cenkalti/backoff"
-  packages = ["."]
-  revision = "61153c768f31ee5f130071d08fc82b85208528de"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/cpuguy83/go-md2man"
-  packages = ["md2man"]
-  revision = "1d903dcb749992f3741d744c0f8376b4bd7eb3e1"
-  version = "v1.0.7"
-
-[[projects]]
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29"
-  version = "v3.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/dustin/go-humanize"
-  packages = ["."]
-  revision = "bb3d318650d48840a39aa21a027c6630e198e626"
-
-[[projects]]
-  name = "github.com/elithrar/simple-scrypt"
-  packages = ["."]
-  revision = "2325946f714c95de4a6088202c402fbdfa64163b"
-  version = "v1.2.0"
-
-[[projects]]
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a"
-  version = "v1.32.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  revision = "c65a0412e71e8b9b3bfd22925720d23c0f054237"
-
-[[projects]]
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  name = "github.com/juju/ratelimit"
-  packages = ["."]
-  revision = "59fac5042749a5afb9af70e813da1dd5474f0167"
-  version = "1.0.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/fs"
-  packages = ["."]
-  revision = "2788f0dbd16903de03cb8186e5c7d97b69ad387b"
-
-[[projects]]
-  name = "github.com/kurin/blazer"
-  packages = ["b2","base","internal/b2types","internal/blog"]
-  revision = "e269a1a17bb6aec278c06a57cb7e8f8d0d333e04"
-  version = "v0.2.1"
-
-[[projects]]
-  name = "github.com/marstr/guid"
-  packages = ["."]
-  revision = "8bdf7d1a087ccc975cf37dd6507da50698fd19ca"
-
-[[projects]]
-  name = "github.com/minio/minio-go"
-  packages = [".","pkg/credentials","pkg/encrypt","pkg/policy","pkg/s3signer","pkg/s3utils","pkg/set"]
-  revision = "14f1d472d115bac5ca4804094aa87484a72ced61"
-  version = "4.0.6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ncw/swift"
-  packages = ["."]
-  revision = "ae9f0ea1605b9aa6434ed5c731ca35d83ba67c55"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pkg/profile"
-  packages = ["."]
-  revision = "5b67d428864e92711fcbd2f8629456121a56d91f"
-  version = "v1.2.1"
-
-[[projects]]
-  name = "github.com/pkg/sftp"
-  packages = ["."]
-  revision = "f6a9258a0f570c3a76681b897b6ded57cb0dfa88"
-  version = "1.2.0"
-
-[[projects]]
-  name = "github.com/pkg/xattr"
-  packages = ["."]
-  revision = "23c75e3f6c1d8b13b3dd905b011a7f38a06044b7"
-  version = "v0.2.1"
-
-[[projects]]
-  name = "github.com/restic/chunker"
-  packages = ["."]
-  revision = "db83917be3b88cc307464b7d8a221c173e34a0db"
-  version = "v0.2.0"
-
-[[projects]]
-  name = "github.com/russross/blackfriday"
-  packages = ["."]
-  revision = "4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c"
-  version = "v1.5"
-
-[[projects]]
-  name = "github.com/satori/go.uuid"
-  packages = ["."]
-  revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
-  version = "v1.2.0"
-
-[[projects]]
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba"
-  version = "v1.0.4"
-
-[[projects]]
-  name = "github.com/spf13/cobra"
-  packages = [".","doc"]
-  revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
-  version = "v0.0.1"
-
-[[projects]]
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = ["curve25519","ed25519","ed25519/internal/edwards25519","internal/chacha20","pbkdf2","poly1305","scrypt","ssh","ssh/terminal"]
-  revision = "3d37316aaa6bd9929127ac9a527abf408178ea7b"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = ["context","context/ctxhttp","idna","lex/httplex"]
-  revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-  packages = [".","google","internal","jws","jwt"]
-  revision = "b28fcf2b08a19742b43084fb40ab78ac6c3d8067"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix","windows"]
-  revision = "af50095a40f9041b3b38960738837185c26e9419"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
-  revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
-
-[[projects]]
-  branch = "master"
-  name = "google.golang.org/api"
-  packages = ["gensupport","googleapi","googleapi/internal/uritemplates","storage/v1"]
-  revision = "65b0d8655182691ad23b4fac11e6f7b897d9b634"
-
-[[projects]]
-  name = "google.golang.org/appengine"
-  packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "v2"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,21 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"

Makefile

@@ -3,7 +3,7 @@
 all: restic
 
 restic:
-	go run build.go
+	go run -mod=vendor build.go || go run build.go
 
 clean:
 	rm -f restic

README.rst

@@ -1,4 +1,4 @@
-|Documentation| |Build Status| |Build status| |Report Card| |Say Thanks| |TestCoverage|
+|Documentation| |Build Status| |Build status| |Report Card| |Say Thanks| |TestCoverage| |Reviewed by Hound|
 
 Introduction
 ------------
@@ -29,7 +29,7 @@ and add some data:
 
 .. code-block:: console
 
-    $ restic -r /tmp/backup backup ~/work
+    $ restic --repo /tmp/backup backup ~/work
     enter password for repository:
     scan [/home/user/work]
     scanned 764 directories, 1816 files in 0:00
@@ -57,6 +57,7 @@ Therefore, restic supports the following backends for storing backups natively:
 - `BackBlaze B2 <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#backblaze-b2>`__
 - `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#microsoft-azure-blob-storage>`__
 - `Google Cloud Storage <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#google-cloud-storage>`__
+- And many other services via the `rclone <https://rclone.org>`__ `Backend <https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#other-services-via-rclone>`__
 
 Design Principles
 -----------------
@@ -110,10 +111,18 @@ License
 Restic is licensed under `BSD 2-Clause License <https://opensource.org/licenses/BSD-2-Clause>`__. You can find the
 complete text in ``LICENSE``.
 
+Sponsorship
+-----------
+
+Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
+Storage are sponsored by `AppsCode <https://appscode.com>`__!
+
+|AppsCode|
+
 .. |Documentation| image:: https://readthedocs.org/projects/restic/badge/?version=latest
    :target: https://restic.readthedocs.io/en/latest/?badge=latest
-.. |Build Status| image:: https://travis-ci.org/restic/restic.svg?branch=master
-   :target: https://travis-ci.org/restic/restic
+.. |Build Status| image:: https://travis-ci.com/restic/restic.svg?branch=master
+   :target: https://travis-ci.com/restic/restic
 .. |Build status| image:: https://ci.appveyor.com/api/projects/status/nuy4lfbgfbytw92q/branch/master?svg=true
    :target: https://ci.appveyor.com/project/fd0/restic/branch/master
 .. |Report Card| image:: https://goreportcard.com/badge/github.com/restic/restic
@@ -122,3 +131,7 @@ complete text in ``LICENSE``.
    :target: https://saythanks.io/to/restic
 .. |TestCoverage| image:: https://codecov.io/gh/restic/restic/branch/master/graph/badge.svg
    :target: https://codecov.io/gh/restic/restic
+.. |AppsCode| image:: https://cdn.appscode.com/images/logo/appscode/ac-logo-color.png
+   :target: https://appscode.com
+.. |Reviewed by Hound| image:: https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg
+   :target: https://houndci.com

VERSION

@@ -1 +1 @@
-0.8.2
+0.9.3

appveyor.yml

@@ -7,6 +7,9 @@ branches:
   only:
     - master
 
+cache:
+  - '%LocalAppData%\go-build'
+
 init:
   - ps: >-
       $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
@@ -17,8 +20,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.10.windows-amd64.msi
-  - msiexec /i go1.10.windows-amd64.msi /q
+  - appveyor DownloadFile https://dl.google.com/go/go1.11.windows-amd64.msi
+  - msiexec /i go1.11.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
@@ -26,4 +29,4 @@ install:
   - set PATH=bin/;%PATH%
 
 build_script:
-  - go run run_integration_tests.go
+  - go run -mod=vendor run_integration_tests.go

build.go

@@ -1,3 +1,18 @@
+// Description
+//
+// This program aims to make building Go programs for end users easier by just
+// calling it with `go run`, without having to setup a GOPATH.
+//
+// For Go < 1.11, it'll create a new GOPATH in a temporary directory, then run
+// `go build` on the package configured as Main in the Config struct.
+//
+// For Go >= 1.11 if the file go.mod is present, it'll use Go modules and not
+// setup a GOPATH. It builds the package configured as Main in the Config
+// struct with `go build -mod=vendor` to use the vendored dependencies.
+// The variable GOPROXY is set to `off` so that no network calls are made. All
+// files are copied to a temporary directory before `go build` is called within
+// that directory.
+
 // BSD 2-Clause License
 //
 // Copyright (c) 2016-2018, Alexander Neumann <alexander@bumpern.de>
@@ -37,7 +52,6 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -46,23 +60,22 @@ import (
 
 // config contains the configuration for the program to build.
 var config = Config{
-	Name:      "restic",                              // name of the program executable and directory
-	Namespace: "github.com/restic/restic",            // subdir of GOPATH, e.g. "github.com/foo/bar"
-	Main:      "github.com/restic/restic/cmd/restic", // package name for the main package
-	Tests: []string{ // tests to run
-		"github.com/restic/restic/internal/...",
-		"github.com/restic/restic/cmd/...",
-	},
-	MinVersion: GoVersion{Major: 1, Minor: 8, Patch: 0}, // minimum Go version supported
+	Name:             "restic",                                // name of the program executable and directory
+	Namespace:        "github.com/restic/restic",              // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Main:             "./cmd/restic",                          // package name for the main package
+	DefaultBuildTags: []string{"selfupdate"},                  // specify build tags which are always used
+	Tests:            []string{"./..."},                       // tests to run
+	MinVersion:       GoVersion{Major: 1, Minor: 9, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
 type Config struct {
-	Name       string
-	Namespace  string
-	Main       string
-	Tests      []string
-	MinVersion GoVersion
+	Name             string
+	Namespace        string
+	Main             string
+	DefaultBuildTags []string
+	Tests            []string
+	MinVersion       GoVersion
 }
 
 var (
@@ -70,41 +83,13 @@ var (
 	keepGopath bool
 	runTests   bool
 	enableCGO  bool
+	enablePIE  bool
+	goVersion  = ParseGoVersion(runtime.Version())
 )
 
-// specialDir returns true if the file begins with a special character ('.' or '_').
-func specialDir(name string) bool {
-	if name == "." {
-		return false
-	}
-
-	base := filepath.Base(name)
-	if base == "vendor" || base[0] == '_' || base[0] == '.' {
-		return true
-	}
-
-	return false
-}
-
-// excludePath returns true if the file should not be copied to the new GOPATH.
-func excludePath(name string) bool {
-	ext := path.Ext(name)
-	if ext == ".go" || ext == ".s" || ext == ".h" {
-		return false
-	}
-
-	parentDir := filepath.Base(filepath.Dir(name))
-	if parentDir == "testdata" {
-		return false
-	}
-
-	return true
-}
-
-// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
-// to dst/prefix/, so calling
+// copy all Go files in src to dst, creating directories on the fly, so calling
 //
-//   updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic")
+//   copy("/tmp/gopath/src/github.com/restic/restic", "/home/u/restic")
 //
 // with "/home/u/restic" containing the file "foo.go" yields the following tree
 // at "/tmp/gopath":
@@ -115,19 +100,15 @@ func excludePath(name string) bool {
 //           └── restic
 //               └── restic
 //                   └── foo.go
-func updateGopath(dst, src, prefix string) error {
-	verbosePrintf("copy contents of %v to %v\n", src, filepath.Join(dst, prefix))
+func copy(dst, src string) error {
+	verbosePrintf("copy contents of %v to %v\n", src, dst)
 	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
 		if name == src {
 			return err
 		}
 
-		if specialDir(name) {
-			if fi.IsDir() {
-				return filepath.SkipDir
-			}
-
-			return nil
+		if name == ".git" {
+			return filepath.SkipDir
 		}
 
 		if err != nil {
@@ -138,17 +119,13 @@ func updateGopath(dst, src, prefix string) error {
 			return nil
 		}
 
-		if excludePath(name) {
-			return nil
-		}
-
 		intermediatePath, err := filepath.Rel(src, name)
 		if err != nil {
 			return err
 		}
 
 		fileSrc := filepath.Join(src, intermediatePath)
-		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)
+		fileDst := filepath.Join(dst, intermediatePath)
 
 		return copyFile(fileDst, fileSrc)
 	})
@@ -163,6 +140,15 @@ func directoryExists(dirname string) bool {
 	return stat.IsDir()
 }
 
+func fileExists(filename string) bool {
+	stat, err := os.Stat(filename)
+	if err != nil && os.IsNotExist(err) {
+		return false
+	}
+
+	return stat.Mode().IsRegular()
+}
+
 // copyFile creates dst from src, preserving file attributes and timestamps.
 func copyFile(dst, src string) error {
 	fi, err := os.Stat(src)
@@ -182,30 +168,34 @@ func copyFile(dst, src string) error {
 
 	fdst, err := os.Create(dst)
 	if err != nil {
+		_ = fsrc.Close()
 		return err
 	}
 
-	if _, err = io.Copy(fdst, fsrc); err != nil {
+	_, err = io.Copy(fdst, fsrc)
+	if err != nil {
+		_ = fsrc.Close()
+		_ = fdst.Close()
 		return err
 	}
 
-	if err == nil {
-		err = fsrc.Close()
+	err = fdst.Close()
+	if err != nil {
+		_ = fsrc.Close()
+		return err
 	}
 
-	if err == nil {
-		err = fdst.Close()
+	err = fsrc.Close()
+	if err != nil {
+		return err
 	}
 
-	if err == nil {
-		err = os.Chmod(dst, fi.Mode())
+	err = os.Chmod(dst, fi.Mode())
+	if err != nil {
+		return err
 	}
 
-	if err == nil {
-		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
-	}
-
-	return nil
+	return os.Chtimes(dst, fi.ModTime(), fi.ModTime())
 }
 
 // die prints the message with fmt.Fprintf() to stderr and exits with an error
@@ -221,13 +211,15 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "OPTIONS:\n")
 	fmt.Fprintf(output, "  -v     --verbose       output more messages\n")
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
-	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
+	fmt.Fprintf(output, "  -k     --keep-tempdir  do not remove the temporary directory after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
 	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --enable-cgo    use CGO to link against libc\n")
+	fmt.Fprintf(output, "         --enable-pie    use PIE buildmode\n")
 	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
-	fmt.Fprintf(output, "         --goarm value  set GOARM for cross-compilation\n")
+	fmt.Fprintf(output, "         --goarm value   set GOARM for cross-compilation\n")
+	fmt.Fprintf(output, "         --tempdir dir   use a specific directory for compilation\n")
 }
 
 func verbosePrintf(message string, args ...interface{}) {
@@ -238,11 +230,20 @@ func verbosePrintf(message string, args ...interface{}) {
 	fmt.Printf("build: "+message, args...)
 }
 
-// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if
-// present).
+// cleanEnv returns a clean environment with GOPATH, GOBIN and GO111MODULE
+// removed (if present).
 func cleanEnv() (env []string) {
+	removeKeys := map[string]struct{}{
+		"GOPATH":      struct{}{},
+		"GOBIN":       struct{}{},
+		"GO111MODULE": struct{}{},
+	}
+
 	for _, v := range os.Environ() {
-		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
+		data := strings.SplitN(v, "=", 2)
+		name := data[0]
+
+		if _, ok := removeKeys[name]; ok {
 			continue
 		}
 
@@ -253,15 +254,27 @@ func cleanEnv() (env []string) {
 }
 
 // build runs "go build args..." with GOPATH set to gopath.
-func build(cwd, goos, goarch, goarm, gopath string, args ...string) error {
+func build(cwd string, env map[string]string, args ...string) error {
 	a := []string{"build"}
-	a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", gopath))
-	a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", gopath))
+
+	if goVersion.AtLeast(GoVersion{1, 10, 0}) {
+		verbosePrintf("Go version is at least 1.10, using new syntax for -gcflags\n")
+		// use new prefix
+		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
+		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
+	} else {
+		a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", cwd))
+		a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", cwd))
+	}
+	if enablePIE {
+		a = append(a, "-buildmode=pie")
+	}
+
 	a = append(a, args...)
 	cmd := exec.Command("go", a...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
-	if goarm != "" {
-		cmd.Env = append(cmd.Env, "GOARM="+goarm)
+	cmd.Env = append(cleanEnv(), "GOPROXY=off")
+	for k, v := range env {
+		cmd.Env = append(cmd.Env, k+"="+v)
 	}
 	if !enableCGO {
 		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
@@ -270,20 +283,30 @@ func build(cwd, goos, goarch, goarm, gopath string, args ...string) error {
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	verbosePrintf("go %s\n", args)
+
+	verbosePrintf("chdir %q\n", cwd)
+	verbosePrintf("go %q\n", a)
 
 	return cmd.Run()
 }
 
 // test runs "go test args..." with GOPATH set to gopath.
-func test(cwd, gopath string, args ...string) error {
-	args = append([]string{"test"}, args...)
+func test(cwd string, env map[string]string, args ...string) error {
+	args = append([]string{"test", "-count", "1"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
+	cmd.Env = append(cleanEnv(), "GOPROXY=off")
+	for k, v := range env {
+		cmd.Env = append(cmd.Env, k+"="+v)
+	}
+	if !enableCGO {
+		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
+	}
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	verbosePrintf("go %s\n", args)
+
+	verbosePrintf("chdir %q\n", cwd)
+	verbosePrintf("go %q\n", args)
 
 	return cmd.Run()
 }
@@ -366,30 +389,39 @@ func ParseGoVersion(s string) (v GoVersion) {
 
 	s = s[2:]
 	data := strings.Split(s, ".")
-	if len(data) != 3 {
-		return
+	if len(data) < 2 || len(data) > 3 {
+		// invalid version
+		return GoVersion{}
 	}
 
-	major, err := strconv.Atoi(data[0])
+	var err error
+
+	v.Major, err = strconv.Atoi(data[0])
 	if err != nil {
-		return
+		return GoVersion{}
 	}
 
-	minor, err := strconv.Atoi(data[1])
-	if err != nil {
-		return
+	// try to parse the minor version while removing an eventual suffix (like
+	// "rc2" or so)
+	for s := data[1]; s != ""; s = s[:len(s)-1] {
+		v.Minor, err = strconv.Atoi(s)
+		if err == nil {
+			break
+		}
 	}
 
-	patch, err := strconv.Atoi(data[2])
-	if err != nil {
-		return
+	if v.Minor == 0 {
+		// no minor version found
+		return GoVersion{}
 	}
 
-	v = GoVersion{
-		Major: major,
-		Minor: minor,
-		Patch: patch,
+	if len(data) >= 3 {
+		v.Patch, err = strconv.Atoi(data[2])
+		if err != nil {
+			return GoVersion{}
+		}
 	}
+
 	return
 }
 
@@ -422,20 +454,24 @@ func (v GoVersion) String() string {
 }
 
 func main() {
-	ver := ParseGoVersion(runtime.Version())
-	if !ver.AtLeast(config.MinVersion) {
-		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", ver, config.MinVersion)
+	if !goVersion.AtLeast(config.MinVersion) {
+		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion)
 		os.Exit(1)
 	}
 
-	buildTags := []string{}
+	buildTags := config.DefaultBuildTags
 
 	skipNext := false
 	params := os.Args[1:]
 
-	targetGOOS := runtime.GOOS
-	targetGOARCH := runtime.GOARCH
-	targetGOARM := ""
+	goEnv := map[string]string{}
+	buildEnv := map[string]string{
+		"GOOS":   runtime.GOOS,
+		"GOARCH": runtime.GOARCH,
+		"GOARM":  "",
+	}
+
+	tempdir := ""
 
 	var outputFilename string
 
@@ -455,23 +491,28 @@ func main() {
 				die("-t given but no tag specified")
 			}
 			skipNext = true
-			buildTags = strings.Split(params[i+1], " ")
+			buildTags = append(buildTags, strings.Split(params[i+1], " ")...)
 		case "-o", "--output":
 			skipNext = true
 			outputFilename = params[i+1]
+		case "--tempdir":
+			skipNext = true
+			tempdir = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--enable-cgo":
 			enableCGO = true
+		case "--enable-pie":
+			enablePIE = true
 		case "--goos":
 			skipNext = true
-			targetGOOS = params[i+1]
+			buildEnv["GOOS"] = params[i+1]
 		case "--goarch":
 			skipNext = true
-			targetGOARCH = params[i+1]
+			buildEnv["GOARCH"] = params[i+1]
 		case "--goarm":
 			skipNext = true
-			targetGOARM = params[i+1]
+			buildEnv["GOARM"] = params[i+1]
 		case "-h":
 			showUsage(os.Stdout)
 			return
@@ -482,10 +523,7 @@ func main() {
 		}
 	}
 
-	if len(buildTags) == 0 {
-		verbosePrintf("adding build-tag release\n")
-		buildTags = []string{"release"}
-	}
+	verbosePrintf("detected Go version %v\n", goVersion)
 
 	for i := range buildTags {
 		buildTags[i] = strings.TrimSpace(buildTags[i])
@@ -498,48 +536,16 @@ func main() {
 		die("Getwd(): %v\n", err)
 	}
 
-	gopath, err := ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
-	if err != nil {
-		die("TempDir(): %v\n", err)
-	}
-
-	verbosePrintf("create GOPATH at %v\n", gopath)
-	if err = updateGopath(gopath, root, config.Namespace); err != nil {
-		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
-	}
-
-	vendor := filepath.Join(root, "vendor")
-	if directoryExists(vendor) {
-		if err = updateGopath(gopath, vendor, filepath.Join(config.Namespace, "vendor")); err != nil {
-			die("copying files from %v to %v failed: %v\n", root, gopath, err)
-		}
-	}
-
-	defer func() {
-		if !keepGopath {
-			verbosePrintf("remove %v\n", gopath)
-			if err = os.RemoveAll(gopath); err != nil {
-				die("remove GOPATH at %s failed: %v\n", err)
-			}
-		} else {
-			verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
-		}
-	}()
-
 	if outputFilename == "" {
 		outputFilename = config.Name
-		if targetGOOS == "windows" {
+		if buildEnv["GOOS"] == "windows" {
 			outputFilename += ".exe"
 		}
 	}
 
-	cwd, err := os.Getwd()
-	if err != nil {
-		die("Getwd() returned %v\n", err)
-	}
 	output := outputFilename
 	if !filepath.IsAbs(output) {
-		output = filepath.Join(cwd, output)
+		output = filepath.Join(root, output)
 	}
 
 	version := getVersion()
@@ -550,13 +556,65 @@ func main() {
 	ldflags := "-s -w " + constants.LDFlags()
 	verbosePrintf("ldflags: %s\n", ldflags)
 
-	args := []string{
-		"-tags", strings.Join(buildTags, " "),
-		"-ldflags", ldflags,
-		"-o", output, config.Main,
+	var (
+		buildArgs []string
+		testArgs  []string
+	)
+
+	mainPackage := config.Main
+	if strings.HasPrefix(mainPackage, config.Namespace) {
+		mainPackage = strings.Replace(mainPackage, config.Namespace, "./", 1)
 	}
 
-	err = build(filepath.Join(gopath, "src"), targetGOOS, targetGOARCH, targetGOARM, gopath, args...)
+	buildTarget := filepath.FromSlash(mainPackage)
+	buildCWD := ""
+
+	if goVersion.AtLeast(GoVersion{1, 11, 0}) && fileExists("go.mod") {
+		verbosePrintf("Go >= 1.11 and 'go.mod' found, building with modules\n")
+		buildCWD = root
+
+		buildArgs = append(buildArgs, "-mod=vendor")
+		testArgs = append(testArgs, "-mod=vendor")
+	} else {
+		if tempdir == "" {
+			tempdir, err = ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
+			if err != nil {
+				die("TempDir(): %v\n", err)
+			}
+		}
+
+		verbosePrintf("Go < 1.11 or 'go.mod' not found, create GOPATH at %v\n", tempdir)
+		targetdir := filepath.Join(tempdir, "src", filepath.FromSlash(config.Namespace))
+		if err = copy(targetdir, root); err != nil {
+			die("copying files from %v to %v/src failed: %v\n", root, tempdir, err)
+		}
+
+		defer func() {
+			if !keepGopath {
+				verbosePrintf("remove %v\n", tempdir)
+				if err = os.RemoveAll(tempdir); err != nil {
+					die("remove GOPATH at %s failed: %v\n", tempdir, err)
+				}
+			} else {
+				verbosePrintf("leaving temporary GOPATH at %v\n", tempdir)
+			}
+		}()
+
+		buildCWD = targetdir
+
+		goEnv["GOPATH"] = tempdir
+		buildEnv["GOPATH"] = tempdir
+	}
+
+	verbosePrintf("environment:\n  go: %v\n  build: %v\n", goEnv, buildEnv)
+
+	buildArgs = append(buildArgs,
+		"-tags", strings.Join(buildTags, " "),
+		"-ldflags", ldflags,
+		"-o", output, buildTarget,
+	)
+
+	err = build(buildCWD, buildEnv, buildArgs...)
 	if err != nil {
 		die("build failed: %v\n", err)
 	}
@@ -564,7 +622,9 @@ func main() {
 	if runTests {
 		verbosePrintf("running tests\n")
 
-		err = test(cwd, gopath, config.Tests...)
+		testArgs = append(testArgs, config.Tests...)
+
+		err = test(buildCWD, goEnv, testArgs...)
 		if err != nil {
 			die("running tests failed: %v\n", err)
 		}

changelog/0.7.1_2017-07-22/issue-1055

@@ -2,9 +2,9 @@ Enhancement: Create subdirs below `data/` for local/sftp backends
 
 The local and sftp backends now create the subdirs below `data/` on
 open/init. This way, restic makes sure that they always exist. This is
-connected to an issue for the sftp server:
+connected to an issue for the sftp server.
 
-https://github.com/restic/rest-server/pull/11#issuecomment-309879710
 https://github.com/restic/restic/issues/1055
+https://github.com/restic/rest-server/pull/11#issuecomment-309879710
 https://github.com/restic/restic/pull/1077
 https://github.com/restic/restic/pull/1105

changelog/0.7.1_2017-07-22/issue-1073

@@ -1,6 +1,6 @@
 Enhancement: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
 
-The `migrate` command for chaning the `s3legacy` layout to the `default` layout
+The `migrate` command for changing the `s3legacy` layout to the `default` layout
 for s3 backends has been improved: It can now be restarted with `restic migrate
 --force s3_layout` and automatically retries operations on error.
 

changelog/0.8.3_2018-02-26/issue-1497

@@ -0,0 +1,8 @@
+Enhancement: Add --read-data-subset flag to check command
+
+This change introduces ability to check integrity of a subset of repository 
+data packs. This can be used to spread integrity check of larger repositories
+over a period of time.
+
+https://github.com/restic/restic/issues/1497
+https://github.com/restic/restic/pull/1556

changelog/0.8.3_2018-02-26/issue-1633

@@ -0,0 +1,7 @@
+Bugfix: Fixed unexpected 'pack file cannot be listed' error
+
+Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune`
+commands failed to read pack files with size of 587, 588, 589 or 590 bytes.
+
+https://github.com/restic/restic/issues/1633
+https://github.com/restic/restic/pull/1635

changelog/0.8.3_2018-02-26/issue-1641

@@ -0,0 +1,10 @@
+Bugfix: Ignore files with invalid names in the repo
+
+The release 0.8.2 introduced a bug: when restic encounters files in the repo
+which do not have a valid name, it tries to load a file with a name of lots of
+zeroes instead of ignoring it. This is now resolved, invalid file names are
+just ignored.
+
+https://github.com/restic/restic/issues/1641
+https://github.com/restic/restic/pull/1643
+https://forum.restic.net/t/help-fixing-repo-no-such-file/485/3

changelog/0.8.3_2018-02-26/pull-1560

@@ -0,0 +1,5 @@
+Enhancement: Retry all repository file download errors
+
+Restic will now retry failed downloads, similar to other operations.
+
+https://github.com/restic/restic/pull/1560

changelog/0.8.3_2018-02-26/pull-1623

@@ -0,0 +1,12 @@
+Enhancement: Don't check for presence of files in the backend before writing
+
+Before, all backend implementations were required to return an error if the
+file that is to be written already exists in the backend. For most backends,
+that means making a request (e.g. via HTTP) and returning an error when the
+file already exists.
+
+This is not accurate, the file could have been created between the HTTP request
+testing for it, and when writing starts, so we've relaxed this requeriment,
+which saves one additional HTTP request per newly added file.
+
+https://github.com/restic/restic/pull/1623

changelog/0.8.3_2018-02-26/pull-1634

@@ -0,0 +1,7 @@
+Enhancement: Upgrade B2 client library, reduce HTTP requests
+
+We've upgraded the B2 client library restic uses to access BackBlaze B2. This
+reduces the number of HTTP requests needed to upload a new file from two to
+one, which should improve throughput to B2.
+
+https://github.com/restic/restic/pull/1634

changelog/0.8.3_2018-02-26/pull-1638

@@ -0,0 +1,16 @@
+Bugfix: Handle errors listing files in the backend
+
+A user reported in the forum that restic completes a backup although a
+concurrent `prune` operation was running. A few error messages were printed,
+but the backup was attempted and completed successfully. No error code was
+returned.
+
+This should not happen: The repository is exclusively locked during `prune`, so
+when `restic backup` is run in parallel, it should abort and return an error
+code instead.
+
+It was found that the bug was in the code introduced only recently, which
+retries a List() operation on the backend should that fail. It is now corrected.
+
+https://github.com/restic/restic/pull/1638
+https://forum.restic.net/t/restic-backup-returns-0-exit-code-when-already-locked/484

changelog/0.9.0_2018-05-21/issue-1433

@@ -0,0 +1,12 @@
+Enhancement: Support UTF-16 encoding and process Byte Order Mark
+
+On Windows, text editors commonly leave a Byte Order Mark at the beginning of
+the file to define which encoding is used (oftentimes UTF-16). We've added code
+to support processing the BOMs in text files, like the exclude files, the
+password file and the file passed via `--files-from`. This does not apply to
+any file being saved in a backup, those are not touched and archived as they
+are.
+
+https://github.com/restic/restic/issues/1433
+https://github.com/restic/restic/issues/1738
+https://github.com/restic/restic/pull/1748

changelog/0.9.0_2018-05-21/issue-1561

@@ -0,0 +1,10 @@
+Enhancement: Allow using rclone to access other services
+
+We've added the ability to use rclone to store backup data on all backends that
+it supports. This was done in collaboration with Nick, the author of rclone.
+You can now use it to first configure a service, then restic manages the rest
+(starting and stopping rclone). For details, please see the manual.
+
+https://github.com/restic/restic/issues/1561
+https://github.com/restic/restic/pull/1657
+https://rclone.org

changelog/0.9.0_2018-05-21/issue-1608

@@ -0,0 +1,7 @@
+Bugfix: Respect time stamp for new backup when reading from stdin
+
+When reading backups from stdin (via `restic backup --stdin`), restic now uses
+the time stamp for the new backup passed in `--time`.
+
+https://github.com/restic/restic/issues/1608
+https://github.com/restic/restic/pull/1703

changelog/0.9.0_2018-05-21/issue-1652

@@ -0,0 +1,9 @@
+Bugfix: Ignore/remove invalid lock files
+
+This corrects a bug introduced recently: When an invalid lock file in the repo
+is encountered (e.g. if the file is empty), the code used to ignore that, but
+now returns the error. Now, invalid files are ignored for the normal lock
+check, and removed when `restic unlock --remove-all` is run.
+
+https://github.com/restic/restic/issues/1652
+https://github.com/restic/restic/pull/1653

changelog/0.9.0_2018-05-21/issue-1665

@@ -0,0 +1,27 @@
+Enhancement: Improve cache handling for `restic check`
+
+For safety reasons, restic does not use a local metadata cache for the `restic
+check` command, so that data is loaded from the repository and restic can check
+it's in good condition. When the cache is disabled, restic will fetch each tiny
+blob needed for checking the integrity using a separate backend request. For
+non-local backends, that will take a long time, and depending on the backend
+(e.g. B2) may also be much more expensive.
+
+This PR adds a few commits which will change the behavior as follows:
+
+ * When `restic check` is called without any additional parameters, it will
+   build a new cache in a temporary directory, which is removed at the end of
+   the check. This way, we'll get readahead for metadata files (so restic will
+   fetch the whole file when the first blob from the file is requested), but
+   all data is freshly fetched from the storage backend. This is the default
+   behavior and will work for almost all users.
+
+ * When `restic check` is called with `--with-cache`, the default on-disc cache
+   is used. This behavior hasn't changed since the cache was introduced.
+
+ * When `--no-cache` is specified, restic falls back to the old behavior, and
+   read all tiny blobs in separate requests.
+
+https://github.com/restic/restic/issues/1665
+https://github.com/restic/restic/issues/1694
+https://github.com/restic/restic/pull/1696

changelog/0.9.0_2018-05-21/issue-1721

@@ -0,0 +1,8 @@
+Enhancement: Add `cache` command to list cache dirs
+
+The command `cache` was added, it allows listing restic's cache directoriers
+together with the last usage. It also allows removing old cache dirs without
+having to access a repo, via `restic cache --cleanup`
+
+https://github.com/restic/restic/issues/1721
+https://github.com/restic/restic/pull/1749

changelog/0.9.0_2018-05-21/issue-1730

@@ -0,0 +1,11 @@
+Bugfix: Ignore sockets for restore
+
+We've received a report and correct the behavior in which the restore code
+aborted restoring a directory when a socket was encountered. Unix domain socket
+files cannot be restored (they are created on the fly once a process starts
+listening). The error handling was corrected, and in addition we're now
+ignoring sockets during restore.
+
+https://github.com/restic/restic/issues/1730
+https://github.com/restic/restic/pull/1731
+

changelog/0.9.0_2018-05-21/issue-1758

@@ -0,0 +1,8 @@
+Enhancement: Allow saving OneDrive folders in Windows
+
+Restic now contains a bugfix to two libraries, which allows saving OneDrive
+folders in Windows. In order to use the newer versions of the libraries, the
+minimal version required to compile restic is now Go 1.9.
+
+https://github.com/restic/restic/issues/1758
+https://github.com/restic/restic/pull/1765

changelog/0.9.0_2018-05-21/issue-549

@@ -0,0 +1,43 @@
+Enhancement: Rework archiver code
+
+The core archiver code and the complementary code for the `backup` command was
+rewritten completely. This resolves very annoying issues such as 549. The first
+backup with this release of restic will likely result in all files being
+re-read locally, so it will take a lot longer. The next backup after that will
+be fast again.
+
+Basically, with the old code, restic took the last path component of each
+to-be-saved file or directory as the top-level file/directory within the
+snapshot. This meant that when called as `restic backup /home/user/foo`, the
+snapshot would contain the files in the directory `/home/user/foo` as `/foo`.
+
+This is not the case any more with the new archiver code. Now, restic works
+very similar to what `tar` does: When restic is called with an absolute path to
+save, then it'll preserve the directory structure within the snapshot. For the
+example above, the snapshot would contain the files in the directory within
+`/home/user/foo` in the snapshot. For relative directories, it only preserves
+the relative path components. So `restic backup user/foo` will save the files
+as `/user/foo` in the snapshot.
+
+While we were at it, the status display and notification system was completely
+rewritten. By default, restic now shows which files are currently read (unless
+`--quiet` is specified) in a multi-line status display.
+
+The `backup` command also gained a new option: `--verbose`. It can be specified
+once (which prints a bit more detail what restic is doing) or twice (which
+prints a line for each file/directory restic encountered, together with some
+statistics).
+
+Another issue that was resolved is the new code only reads two files at most.
+The old code would read way too many files in parallel, thereby slowing down
+the backup process on spinning discs a lot.
+
+https://github.com/restic/restic/issues/549
+https://github.com/restic/restic/issues/1286
+https://github.com/restic/restic/issues/446
+https://github.com/restic/restic/issues/1344
+https://github.com/restic/restic/issues/1416
+https://github.com/restic/restic/issues/1456
+https://github.com/restic/restic/issues/1145
+https://github.com/restic/restic/issues/1160
+https://github.com/restic/restic/pull/1494

changelog/0.9.0_2018-05-21/pull-1552

@@ -0,0 +1,13 @@
+Enhancement: Use Google Application Default credentials
+
+Google provide libraries to generate appropriate credentials with various
+fallback sources. This change uses the library to generate our GCS client, which
+allows us to make use of these extra methods.
+
+This should be backward compatible with previous restic behaviour while adding
+the additional capabilities to auth from Google's internal metadata endpoints.
+For users running restic in GCP this can make authentication far easier than it
+was before.
+
+https://github.com/restic/restic/pull/1552
+https://developers.google.com/identity/protocols/application-default-credentials

changelog/0.9.0_2018-05-21/pull-1647

@@ -0,0 +1,9 @@
+Enhancement: Accept AWS_SESSION_TOKEN for the s3 backend
+
+Before, it was not possible to use s3 backend with AWS temporary security
+credentials(with AWS_SESSION_TOKEN). This change gives higher priority to
+credentials.EnvAWS credentials provider.
+
+https://github.com/restic/restic/issues/1477
+https://github.com/restic/restic/pull/1479
+https://github.com/restic/restic/pull/1647

changelog/0.9.0_2018-05-21/pull-1648

@@ -0,0 +1,6 @@
+Enhancement: Ignore AWS permission denied error when creating a repository
+
+It's not possible to use s3 backend scoped to a subdirectory(with specific permissions).
+Restic doesn't try to create repository in a subdirectory, when 'bucket exists' of parent directory check fails due to permission issues.
+
+https://github.com/restic/restic/pull/1648

changelog/0.9.0_2018-05-21/pull-1649

@@ -0,0 +1,3 @@
+Enhancement: Add illumos/Solaris support
+
+https://github.com/restic/restic/pull/1649

changelog/0.9.0_2018-05-21/pull-1684

@@ -0,0 +1,6 @@
+Bugfix: Fix backend tests for rest-server
+
+The REST server for restic now requires an explicit parameter (`--no-auth`) if
+no authentication should be allowed. This is fixed in the tests.
+
+https://github.com/restic/restic/pull/1684

changelog/0.9.0_2018-05-21/pull-1709

@@ -0,0 +1,7 @@
+Enhancement: Improve messages `restic check` prints
+
+Some messages `restic check` prints are not really errors, so from now on
+restic does not treat them as errors any more and exits cleanly.
+
+https://github.com/restic/restic/pull/1709
+https://forum.restic.net/t/what-is-the-standard-procedure-to-follow-if-a-backup-or-restore-is-interrupted/571/2

changelog/0.9.0_2018-05-21/pull-1720

@@ -0,0 +1,7 @@
+Enhancement: Add --new-password-file flag for non-interactive password changes
+
+This makes it possible to change a repository password without being prompted.
+
+https://github.com/restic/restic/issues/827
+https://github.com/restic/restic/pull/1720
+https://forum.restic.net/t/changing-repo-password-without-prompt/591

changelog/0.9.0_2018-05-21/pull-1735

@@ -0,0 +1,9 @@
+Enhancement: Allow keeping a time range of snaphots
+
+We've added the `--keep-within` option to the `forget` command. It instructs
+restic to keep all snapshots within the given duration since the newest
+snapshot. For example, running `restic forget --keep-within 5m7d` will keep all
+snapshots which have been made in the five months and seven days since the
+latest snapshot.
+
+https://github.com/restic/restic/pull/1735

changelog/0.9.0_2018-05-21/pull-1746

@@ -0,0 +1,7 @@
+Bugfix: Correctly parse the argument to --tls-client-cert
+
+Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded)
+instead of the argument that was passed to it. This has been corrected.
+
+https://github.com/restic/restic/issues/1745
+https://github.com/restic/restic/pull/1746

changelog/0.9.0_2018-05-21/pull-1782

@@ -0,0 +1,7 @@
+Enhancement: Use default AWS credentials chain for S3 backend
+
+Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials),
+and reorders the credentials chain for the S3 backend to match AWS's standard,
+which is static credentials, env vars, credentials file, and finally remote.
+
+https://github.com/restic/restic/pull/1782

changelog/0.9.1_2018-06-10/issue-1801

@@ -0,0 +1,9 @@
+Bugfix: Add limiting bandwidth to the rclone backend
+
+The rclone backend did not respect `--limit-upload` or `--limit-download`.
+Oftentimes it's not necessary to use this, as the limiting in rclone itself
+should be used because it gives much better results, but in case a remote
+instance of rclone is used (e.g. called via ssh), it is still relevant to limit
+the bandwidth from restic to rclone.
+
+https://github.com/restic/restic/issues/1801

changelog/0.9.1_2018-06-10/issue-1822

@@ -0,0 +1,9 @@
+Bugfix: Allow uploading large files to MS Azure
+
+Sometimes, restic creates files to be uploaded to the repository which are
+quite large, e.g. when saving directories with many entries or very large
+files. The MS Azure API does not allow uploading files larger that 256MiB
+directly, rather restic needs to upload them in blocks of 100MiB. This is now
+implemented.
+
+https://github.com/restic/restic/issues/1822

changelog/0.9.1_2018-06-10/issue-1825

@@ -0,0 +1,12 @@
+Bugfix: Correct `find` to not skip snapshots
+
+Under certain circumstances, the `find` command was found to skip snapshots
+containing directories with files to look for when the directories haven't been
+modified at all, and were already printed as part of a different snapshot. This
+is now corrected.
+
+In addition, we've switched to our own matching/pattern implementation, so now
+things like `restic find "/home/user/foo/**/main.go"` are possible.
+
+https://github.com/restic/restic/issues/1825
+https://github.com/restic/restic/issues/1823

changelog/0.9.1_2018-06-10/issue-1833

@@ -0,0 +1,9 @@
+Bugfix: Fix caching files on error
+
+During `check` it may happen that different threads access the same file in the
+backend, which is then downloaded into the cache only once. When that fails,
+only the thread which is responsible for downloading the file signals the
+correct error. The other threads just assume that the file has been downloaded
+successfully and then get an error when they try to access the cached file.
+
+https://github.com/restic/restic/issues/1833

changelog/0.9.1_2018-06-10/issue-1834

@@ -0,0 +1,8 @@
+Bugfix: Resolve deadlock
+
+When the "scanning" process restic runs to find out how much data there is does
+not finish before the backup itself is done, restic stops doing anything. This
+is resolved now.
+
+https://github.com/restic/restic/issues/1834
+https://github.com/restic/restic/pull/1835

changelog/0.9.2_2018-08-06/issue-1854

@@ -0,0 +1,16 @@
+Bugfix: Allow saving files/dirs on different fs with `--one-file-system`
+
+restic now allows saving files/dirs on a different file system in a subdir
+correctly even when `--one-file-system` is specified. 
+
+The first thing the restic archiver code does is to build a tree of the target
+files/directories. If it detects that a parent directory is already included
+(e.g. `restic backup /foo /foo/bar/baz`), it'll ignore the latter argument.
+
+Without `--one-file-system`, that's perfectly valid: If `/foo` is to be
+archived, it will include `/foo/bar/baz`. But with `--one-file-system`,
+`/foo/bar/baz` may reside on a different file system, so it won't be included
+with `/foo`.
+
+https://github.com/restic/restic/issues/1854
+https://github.com/restic/restic/pull/1855

changelog/0.9.2_2018-08-06/issue-1870

@@ -0,0 +1,6 @@
+Bugfix: Fix restore with --include
+
+We fixed a bug which prevented restic to restore files with an include filter.
+
+https://github.com/restic/restic/issues/1870
+https://github.com/restic/restic/pull/1900

changelog/0.9.2_2018-08-06/issue-1880

@@ -0,0 +1,12 @@
+Bugfix: Use `--cache-dir` argument for `check` command
+
+`check` command now uses a temporary sub-directory of the specified directory
+if set using the `--cache-dir` argument. If not set, the cache directory is
+created in the default temporary directory as before.
+In either case a temporary cache is used to ensure the actual repository is
+checked (rather than a local copy).
+
+The `--cache-dir` argument was not used by the `check` command, instead a
+cache directory was created in the temporary directory.
+
+https://github.com/restic/restic/issues/1880

changelog/0.9.2_2018-08-06/issue-1893

@@ -0,0 +1,8 @@
+Bugfix: Return error when exclude file cannot be read
+
+A bug was found: when multiple exclude files were passed to restic and one of
+them could not be read, an error was printed and restic continued, ignoring
+even the existing exclude files. Now, an error message is printed and restic
+aborts when an exclude file cannot be read.
+
+https://github.com/restic/restic/issues/1893

changelog/0.9.2_2018-08-06/issue-1906

@@ -0,0 +1,8 @@
+Enhancement: Add support for B2 application keys
+
+Restic can now use so-called "application keys" which can be created in the B2
+dashboard and were only introduced recently. In contrast to the "master key",
+such keys can be restricted to a specific bucket and/or path.
+
+https://github.com/restic/restic/issues/1906
+https://github.com/restic/restic/pull/1914

changelog/0.9.2_2018-08-06/pull-1729

@@ -0,0 +1,4 @@
+Enhancement: Add stats command to get information about a repository
+
+https://github.com/restic/restic/issues/874
+https://github.com/restic/restic/pull/1729

changelog/0.9.2_2018-08-06/pull-1772

@@ -0,0 +1,6 @@
+Enhancement: Add restore --verify to verify restored file content
+
+Restore will print error message if restored file content does not match
+expected SHA256 checksum
+
+https://github.com/restic/restic/pull/1772

changelog/0.9.2_2018-08-06/pull-1853

@@ -0,0 +1,6 @@
+Enhancement: Add JSON output support to `restic key list`
+
+This PR enables users to get the output of `restic key list` in JSON in addition
+to the existing table format.
+
+https://github.com/restic/restic/pull/1853

changelog/0.9.2_2018-08-06/pull-1861

@@ -0,0 +1,6 @@
+Bugfix: Fix case-insensitive search with restic find
+
+We've fixed the behavior for `restic find -i PATTERN`, which was
+broken in v0.9.1.
+
+https://github.com/restic/restic/pull/1861

changelog/0.9.2_2018-08-06/pull-1882

@@ -0,0 +1,8 @@
+Enhancement: S3 backend: accept AWS_SESSION_TOKEN
+
+Before, it was not possible to use s3 backend with AWS temporary security credentials(with AWS_SESSION_TOKEN).
+This change gives higher priority to credentials.EnvAWS credentials provider.
+
+https://github.com/restic/restic/issues/1477
+https://github.com/restic/restic/pull/1479
+https://github.com/restic/restic/pull/1647

changelog/0.9.2_2018-08-06/pull-1901

@@ -0,0 +1,9 @@
+Enhancement: Update the Backblaze B2 library
+
+We've updated the library we're using for accessing the Backblaze B2 service to
+0.5.0 to include support for upcoming so-called "application keys". With this
+feature, you can create access credentials for B2 which are restricted to e.g.
+a single bucket or even a sub-directory of a bucket.
+
+https://github.com/restic/restic/pull/1901
+https://github.com/kurin/blazer

changelog/0.9.3_2018-10-13/issue-1766

@@ -0,0 +1,7 @@
+Enhancement: restore: suppress lchown errors when not running as root
+
+Like "cp" and "rsync" do, restic now only reports errors for changing
+the ownership of files during restore if it is run as root, on non-Windows
+operating systems. On Windows, the error is reported as usual.
+
+https://github.com/restic/restic/issues/1766

changelog/0.9.3_2018-10-13/issue-1909

@@ -0,0 +1,14 @@
+Enhancement: Reject files/dirs by name first
+
+The current scanner/archiver code had an architectural limitation: it always
+ran the `lstat()` system call on all files and directories before a decision to
+include/exclude the file/dir was made. This lead to a lot of unnecessary system
+calls for items that could have been rejected by their name or path only.
+
+We've changed the archiver/scanner implementation so that it now first rejects
+by name/path, and only runs the system call on the remaining items. This
+reduces the number of `lstat()` system calls a lot (depending on the exclude
+settings).
+
+https://github.com/restic/restic/issues/1909
+https://github.com/restic/restic/pull/1912

changelog/0.9.3_2018-10-13/issue-1935

@@ -0,0 +1,8 @@
+Bugfix: Remove truncated files from cache
+
+When a file in the local cache is truncated, and restic tries to access data
+beyond the end of the (cached) file, it used to return an error "EOF". This is
+now fixed, such truncated files are removed and the data is fetched directly
+from the backend.
+
+https://github.com/restic/restic/issues/1935

changelog/0.9.3_2018-10-13/issue-1941

@@ -0,0 +1,15 @@
+Enhancement: Add directory filter to ls command
+
+The ls command can now be filtered by directories, so that only files in the
+given directories will be shown. If the --recursive flag is specified, then
+ls will traverse subfolders and list their files as well.
+
+It used to be possible to specify multiple snapshots, but that has been
+replaced by only one snapshot and the possibility of specifying multiple
+directories.
+
+Specifying directories constrains the walk, which can significantly speed up
+the listing.
+
+https://github.com/restic/restic/issues/1940
+https://github.com/restic/restic/pull/1941

changelog/0.9.3_2018-10-13/issue-1967

@@ -0,0 +1,7 @@
+Enhancement: Use `--host` everywhere
+
+We now use the flag `--host` for all commands which need a host name, using
+`--hostname` (e.g. for `restic backup`) still works, but will print a
+deprecation warning. Also, add the short option `-H` where possible.
+
+https://github.com/restic/restic/issues/1967

changelog/0.9.3_2018-10-13/issue-1978

@@ -0,0 +1,12 @@
+Bugfix: Do not return an error when the scanner is faster than backup
+
+When restic makes a backup, there's a background task called "scanner" which
+collects information on how many files and directories are to be saved, in
+order to display progress information to the user. When the backup finishes
+faster than the scanner, it is aborted because the result is not needed any
+more. This logic contained a bug, where quitting the scanner process was
+treated as an error, and caused restic to print an unhelpful error message
+("context canceled").
+
+https://github.com/restic/restic/issues/1978
+https://github.com/restic/restic/pull/1991

changelog/0.9.3_2018-10-13/issue-2028

@@ -0,0 +1,7 @@
+Enhancement: Display size of cache directories
+
+The `cache` command now by default shows the size of the individual cache
+directories. It can be disabled with `--no-size`.
+
+https://github.com/restic/restic/issues/2028
+https://github.com/restic/restic/pull/2033

changelog/0.9.3_2018-10-13/pull-1780

@@ -0,0 +1,13 @@
+Enhancement: Improve the `find` command
+
+We've updated the `find` command to support multiple patterns.
+
+`restic find` is now able to list the snapshots containing a specific tree
+or blob, or even the snapshots that contain blobs belonging to a given pack.
+A list of IDs can be given, as long as they all have the same type.
+
+The command `find` can also display the pack IDs the blobs belong to, if
+the `--show-pack-id` flag is provided.
+
+https://github.com/restic/restic/issues/1777
+https://github.com/restic/restic/pull/1780

changelog/0.9.3_2018-10-13/pull-1876

@@ -0,0 +1,7 @@
+Enhancement: Display reason why forget keeps snapshots
+
+We've added a column to the list of snapshots `forget` keeps which details the
+reasons to keep a particuliar snapshot. This makes debugging policies for
+forget much easier. Please remember to always try things out with `--dry-run`!
+
+https://github.com/restic/restic/pull/1876

changelog/0.9.3_2018-10-13/pull-1891

@@ -0,0 +1,7 @@
+Enhancement: Accept glob in paths loaded via --files-from
+
+Before that, behaviour was different if paths were appended to command line or
+from a file, because wild card characters were expanded by shell if appended to
+command line, but not expanded if loaded from file.
+
+https://github.com/restic/restic/issues/1891

changelog/0.9.3_2018-10-13/pull-1920

@@ -0,0 +1,8 @@
+Enhancement: Vendor dependencies with Go 1.11 Modules
+
+Until now, we've used `dep` for managing dependencies, we've now switch to
+using Go modules. For users this does not change much, only if you want to
+compile restic without downloading anything with Go 1.11, then you need to run:
+`go build -mod=vendor build.go`
+
+https://github.com/restic/restic/pull/1920

changelog/0.9.3_2018-10-13/pull-1949

@@ -0,0 +1,15 @@
+Enhancement: Add new command `self-update`
+
+We have added a new command called `self-update` which downloads the
+latest released version of restic from GitHub and replaces the current
+binary with it. It does not rely on any external program (so it'll work
+everywhere), but still verifies the GPG signature using the embedded GPG
+public key.
+
+By default, the `self-update` command is hidden behind the `selfupdate`
+built tag, which is only set when restic is built using `build.go` (including
+official releases). The reason for this is that downstream distributions will
+then not include the command by default, so users are encouraged to use the
+platform-specific distribution mechanism.
+
+https://github.com/restic/restic/pull/1949

changelog/0.9.3_2018-10-13/pull-1953

@@ -0,0 +1,7 @@
+Enhancement: ls: Add JSON output support for restic ls cmd
+
+We've implemented listing files in the repository with JSON as output, just
+pass `--json` as an option to `restic ls`. This makes the output of the command
+machine readable.
+
+https://github.com/restic/restic/pull/1953

changelog/CHANGELOG.tmpl

@@ -18,11 +18,11 @@ Details
 {{ range $par := .Paragraphs }}
    {{ wrap $par 80 3 }}
 {{ end -}}
-{{ range $id := .Issues }}
-   https://github.com/restic/restic/issues/{{ $id -}}
+{{ range $url := .IssueURLs }}
+   {{ $url -}}
 {{ end -}}
-{{ range $id := .PRs }}
-   https://github.com/restic/restic/pull/{{ $id -}}
+{{ range $url := .PRURLs }}
+   {{ $url -}}
 {{ end -}}
 {{ range $url := .OtherURLs }}
    {{ $url -}}

changelog/changelog-github.tmpl

@@ -2,8 +2,7 @@
 Changelog for restic {{ .Version }} ({{ .Date }})
 =======================================
 
-The following sections list the changes in restic {{ .Version }} relevant to
-restic users. The changes are ordered by importance.
+The following sections list the changes in restic {{ .Version }} relevant to restic users. The changes are ordered by importance.
 
 Summary
 -------

cmd/restic/background.go

@@ -1,9 +0,0 @@
-// +build !linux
-
-package main
-
-// IsProcessBackground should return true if it is running in the background or false if not
-func IsProcessBackground() bool {
-	//TODO: Check if the process are running in the background in other OS than linux
-	return false
-}

cmd/restic/cleanup.go

@@ -64,7 +64,10 @@ func CleanupHandler(c <-chan os.Signal) {
 		fmt.Fprintf(stderr, "%ssignal %v received, cleaning up\n", ClearLine(), s)
 
 		code := 0
-		if s != syscall.SIGINT {
+
+		if s == syscall.SIGINT {
+			code = 130
+		} else {
 			code = 1
 		}
 

cmd/restic/cmd_backup.go

@@ -2,21 +2,28 @@ package main
 
 import (
 	"bufio"
+	"bytes"
+	"context"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"os"
-	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
+	tomb "gopkg.in/tomb.v2"
 
 	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/textfile"
+	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 var cmdBackup = &cobra.Command{
@@ -27,13 +34,13 @@ The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		if backupOptions.Hostname == "" {
+		if backupOptions.Host == "" {
 			hostname, err := os.Hostname()
 			if err != nil {
 				debug.Log("os.Hostname() returned err: %v", err)
 				return
 			}
-			backupOptions.Hostname = hostname
+			backupOptions.Host = hostname
 		}
 	},
 	DisableAutoGenTag: true,
@@ -42,11 +49,16 @@ given as the arguments.
 			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
 		}
 
-		if backupOptions.Stdin {
-			return readBackupFromStdin(backupOptions, globalOptions, args)
-		}
+		var t tomb.Tomb
+		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
+		t.Go(func() error { term.Run(t.Context(globalOptions.ctx)); return nil })
 
-		return runBackup(backupOptions, globalOptions, args)
+		err := runBackup(backupOptions, globalOptions, term, args)
+		if err != nil {
+			return err
+		}
+		t.Kill(nil)
+		return t.Wait()
 	},
 }
 
@@ -62,7 +74,7 @@ type BackupOptions struct {
 	Stdin            bool
 	StdinFilename    string
 	Tags             []string
-	Hostname         string
+	Host             string
 	FilesFrom        string
 	TimeStamp        string
 	WithAtime        bool
@@ -84,133 +96,16 @@ func init() {
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
 	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
-	f.StringVar(&backupOptions.Hostname, "hostname", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+
+	f.StringVar(&backupOptions.Host, "host", "H", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
+	f.MarkDeprecated("hostname", "use --host")
+
 	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
 	f.StringVar(&backupOptions.TimeStamp, "time", "", "time of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 }
 
-func newScanProgress(gopts GlobalOptions) *restic.Progress {
-	if gopts.Quiet {
-		return nil
-	}
-
-	p := restic.NewProgress()
-	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
-		if IsProcessBackground() {
-			return
-		}
-
-		PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
-	}
-
-	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
-		PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
-	}
-
-	return p
-}
-
-func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
-	if gopts.Quiet {
-		return nil
-	}
-
-	archiveProgress := restic.NewProgress()
-
-	var bps, eta uint64
-	itemsTodo := todo.Files + todo.Dirs
-
-	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
-		if IsProcessBackground() {
-			return
-		}
-
-		sec := uint64(d / time.Second)
-		if todo.Bytes > 0 && sec > 0 && ticker {
-			bps = s.Bytes / sec
-			if s.Bytes >= todo.Bytes {
-				eta = 0
-			} else if bps > 0 {
-				eta = (todo.Bytes - s.Bytes) / bps
-			}
-		}
-
-		itemsDone := s.Files + s.Dirs
-
-		status1 := fmt.Sprintf("[%s] %s  %s / %s  %d / %d items  %d errors  ",
-			formatDuration(d),
-			formatPercent(s.Bytes, todo.Bytes),
-			formatBytes(s.Bytes), formatBytes(todo.Bytes),
-			itemsDone, itemsTodo,
-			s.Errors)
-		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
-
-		if w := stdoutTerminalWidth(); w > 0 {
-			maxlen := w - len(status2) - 1
-
-			if maxlen < 4 {
-				status1 = ""
-			} else if len(status1) > maxlen {
-				status1 = status1[:maxlen-4]
-				status1 += "... "
-			}
-		}
-
-		PrintProgress("%s%s", status1, status2)
-	}
-
-	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
-		fmt.Printf("\nduration: %s\n", formatDuration(d))
-	}
-
-	return archiveProgress
-}
-
-func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
-	if gopts.Quiet {
-		return nil
-	}
-
-	archiveProgress := restic.NewProgress()
-
-	var bps uint64
-
-	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
-		if IsProcessBackground() {
-			return
-		}
-
-		sec := uint64(d / time.Second)
-		if s.Bytes > 0 && sec > 0 && ticker {
-			bps = s.Bytes / sec
-		}
-
-		status1 := fmt.Sprintf("[%s] %s  %s/s", formatDuration(d),
-			formatBytes(s.Bytes),
-			formatBytes(bps))
-
-		if w := stdoutTerminalWidth(); w > 0 {
-			maxlen := w - len(status1)
-
-			if maxlen < 4 {
-				status1 = ""
-			} else if len(status1) > maxlen {
-				status1 = status1[:maxlen-4]
-				status1 += "... "
-			}
-		}
-
-		PrintProgress("%s", status1)
-	}
-
-	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
-		fmt.Printf("\nduration: %s\n", formatDuration(d))
-	}
-
-	return archiveProgress
-}
-
 // filterExisting returns a slice of all existing items, or an error if no
 // items exist at all.
 func filterExisting(items []string) (result []string, err error) {
@@ -231,78 +126,33 @@ func filterExisting(items []string) (result []string, err error) {
 	return
 }
 
-func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
-	if len(args) != 0 {
-		return errors.Fatal("when reading from stdin, no additional files can be specified")
-	}
-
-	fn := opts.StdinFilename
-
-	if fn == "" {
-		return errors.Fatal("filename for backup from stdin must not be empty")
-	}
-
-	if filepath.Base(fn) != fn || path.Base(fn) != fn {
-		return errors.Fatal("filename is invalid (may not contain a directory, slash or backslash)")
-	}
-
-	if gopts.password == "" {
-		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
-	}
-
-	repo, err := OpenRepository(gopts)
-	if err != nil {
-		return err
-	}
-
-	lock, err := lockRepo(repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	err = repo.LoadIndex(gopts.ctx)
-	if err != nil {
-		return err
-	}
-
-	r := &archiver.Reader{
-		Repository: repo,
-		Tags:       opts.Tags,
-		Hostname:   opts.Hostname,
-	}
-
-	_, id, err := r.Archive(gopts.ctx, fn, os.Stdin, newArchiveStdinProgress(gopts))
-	if err != nil {
-		return err
-	}
-
-	Verbosef("archived as %v\n", id.Str())
-	return nil
-}
-
-// readFromFile will read all lines from the given filename and write them to a
-// string array, if filename is empty readFromFile returns and empty string
-// array. If filename is a dash (-), readFromFile will read the lines from
-// the standard input.
+// readFromFile will read all lines from the given filename and return them as
+// a string array, if filename is empty readFromFile returns and empty string
+// array. If filename is a dash (-), readFromFile will read the lines from the
+// standard input.
 func readLinesFromFile(filename string) ([]string, error) {
 	if filename == "" {
 		return nil, nil
 	}
 
-	var r io.Reader = os.Stdin
-	if filename != "-" {
-		f, err := os.Open(filename)
-		if err != nil {
-			return nil, err
-		}
-		defer f.Close()
-		r = f
+	var (
+		data []byte
+		err  error
+	)
+
+	if filename == "-" {
+		data, err = ioutil.ReadAll(os.Stdin)
+	} else {
+		data, err = textfile.Read(filename)
+	}
+
+	if err != nil {
+		return nil, err
 	}
 
 	var lines []string
 
-	scanner := bufio.NewScanner(r)
+	scanner := bufio.NewScanner(bytes.NewReader(data))
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
 		// ignore empty lines
@@ -323,56 +173,49 @@ func readLinesFromFile(filename string) ([]string, error) {
 	return lines, nil
 }
 
-func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
+// Check returns an error when an invalid combination of options was set.
+func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 	if opts.FilesFrom == "-" && gopts.password == "" {
 		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 	}
 
-	fromfile, err := readLinesFromFile(opts.FilesFrom)
-	if err != nil {
-		return err
-	}
-
-	// merge files from files-from into normal args so we can reuse the normal
-	// args checks and have the ability to use both files-from and args at the
-	// same time
-	args = append(args, fromfile...)
-	if len(args) == 0 {
-		return errors.Fatal("nothing to backup, please specify target files/dirs")
-	}
-
-	target := make([]string, 0, len(args))
-	for _, d := range args {
-		if a, err := filepath.Abs(d); err == nil {
-			d = a
+	if opts.Stdin {
+		if opts.FilesFrom != "" {
+			return errors.Fatal("--stdin and --files-from cannot be used together")
+		}
+
+		if len(args) > 0 {
+			return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
 		}
-		target = append(target, d)
 	}
 
-	target, err = filterExisting(target)
-	if err != nil {
-		return err
-	}
+	return nil
+}
 
-	// rejectFuncs collect functions that can reject items from the backup
-	var rejectFuncs []RejectFunc
-
-	// allowed devices
-	if opts.ExcludeOtherFS {
-		f, err := rejectByDevice(target)
+// collectRejectByNameFuncs returns a list of all functions which may reject data
+// from being saved in a snapshot based on path only
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectByNameFunc, err error) {
+	// exclude restic cache
+	if repo.Cache != nil {
+		f, err := rejectResticCache(repo)
 		if err != nil {
-			return err
+			return nil, err
 		}
-		rejectFuncs = append(rejectFuncs, f)
+
+		fs = append(fs, f)
 	}
 
 	// add patterns from file
 	if len(opts.ExcludeFiles) > 0 {
-		opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...)
+		excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
+		if err != nil {
+			return nil, err
+		}
+		opts.Excludes = append(opts.Excludes, excludes...)
 	}
 
 	if len(opts.Excludes) > 0 {
-		rejectFuncs = append(rejectFuncs, rejectByPattern(opts.Excludes))
+		fs = append(fs, rejectByPattern(opts.Excludes))
 	}
 
 	if opts.ExcludeCaches {
@@ -382,124 +225,52 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	for _, spec := range opts.ExcludeIfPresent {
 		f, err := rejectIfPresent(spec)
 		if err != nil {
-			return err
+			return nil, err
 		}
 
-		rejectFuncs = append(rejectFuncs, f)
+		fs = append(fs, f)
 	}
 
-	repo, err := OpenRepository(gopts)
-	if err != nil {
-		return err
-	}
-
-	lock, err := lockRepo(repo)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	// exclude restic cache
-	if repo.Cache != nil {
-		f, err := rejectResticCache(repo)
-		if err != nil {
-			return err
-		}
-
-		rejectFuncs = append(rejectFuncs, f)
-	}
-
-	err = repo.LoadIndex(gopts.ctx)
-	if err != nil {
-		return err
-	}
-
-	var parentSnapshotID *restic.ID
-
-	// Force using a parent
-	if !opts.Force && opts.Parent != "" {
-		id, err := restic.FindSnapshot(repo, opts.Parent)
-		if err != nil {
-			return errors.Fatalf("invalid id %q: %v", opts.Parent, err)
-		}
-
-		parentSnapshotID = &id
-	}
-
-	// Find last snapshot to set it as parent, if not already set
-	if !opts.Force && parentSnapshotID == nil {
-		id, err := restic.FindLatestSnapshot(gopts.ctx, repo, target, []restic.TagList{}, opts.Hostname)
-		if err == nil {
-			parentSnapshotID = &id
-		} else if err != restic.ErrNoSnapshotFound {
-			return err
-		}
-	}
-
-	if parentSnapshotID != nil {
-		Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
-	}
-
-	Verbosef("scan %v\n", target)
-
-	selectFilter := func(item string, fi os.FileInfo) bool {
-		for _, reject := range rejectFuncs {
-			if reject(item, fi) {
-				return false
-			}
-		}
-		return true
-	}
-
-	stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts))
-	if err != nil {
-		return err
-	}
-
-	arch := archiver.New(repo)
-	arch.Excludes = opts.Excludes
-	arch.SelectFilter = selectFilter
-	arch.WithAccessTime = opts.WithAtime
-
-	arch.Warn = func(dir string, fi os.FileInfo, err error) {
-		// TODO: make ignoring errors configurable
-		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
-	}
-
-	timeStamp := time.Now()
-	if opts.TimeStamp != "" {
-		timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
-		if err != nil {
-			return errors.Fatalf("error in time option: %v\n", err)
-		}
-	}
-
-	_, id, err := arch.Snapshot(gopts.ctx, newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID, timeStamp)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("snapshot %s saved\n", id.Str())
-
-	return nil
+	return fs, nil
 }
 
-func readExcludePatternsFromFiles(excludeFiles []string) []string {
+// collectRejectFuncs returns a list of all functions which may reject data
+// from being saved in a snapshot based on path and file info
+func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
+	// allowed devices
+	if opts.ExcludeOtherFS && !opts.Stdin {
+		f, err := rejectByDevice(targets)
+		if err != nil {
+			return nil, err
+		}
+		fs = append(fs, f)
+	}
+
+	return fs, nil
+}
+
+// readExcludePatternsFromFiles reads all exclude files and returns the list of
+// exclude patterns. For each line, leading and trailing white space is removed
+// and comment lines are ignored. For each remaining pattern, environment
+// variables are resolved. For adding a literal dollar sign ($), write $$ to
+// the file.
+func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
+	getenvOrDollar := func(s string) string {
+		if s == "$" {
+			return "$"
+		}
+		return os.Getenv(s)
+	}
+
 	var excludes []string
 	for _, filename := range excludeFiles {
 		err := func() (err error) {
-			file, err := fs.Open(filename)
+			data, err := textfile.Read(filename)
 			if err != nil {
 				return err
 			}
-			defer func() {
-				// return pre-close error if there was one
-				if errClose := file.Close(); err == nil {
-					err = errClose
-				}
-			}()
 
-			scanner := bufio.NewScanner(file)
+			scanner := bufio.NewScanner(bytes.NewReader(data))
 			for scanner.Scan() {
 				line := strings.TrimSpace(scanner.Text())
 
@@ -513,15 +284,256 @@ func readExcludePatternsFromFiles(excludeFiles []string) []string {
 					continue
 				}
 
-				line = os.ExpandEnv(line)
+				line = os.Expand(line, getenvOrDollar)
 				excludes = append(excludes, line)
 			}
 			return scanner.Err()
 		}()
 		if err != nil {
-			Warnf("error reading exclude patterns: %v:", err)
-			return nil
+			return nil, err
 		}
 	}
-	return excludes
+	return excludes, nil
+}
+
+// collectTargets returns a list of target files/dirs from several sources.
+func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
+	if opts.Stdin {
+		return nil, nil
+	}
+
+	fromfile, err := readLinesFromFile(opts.FilesFrom)
+	if err != nil {
+		return nil, err
+	}
+
+	// expand wildcards
+	var lines []string
+	for _, line := range fromfile {
+		var expanded []string
+		expanded, err := filepath.Glob(line)
+		if err != nil {
+			return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
+		}
+		lines = append(lines, expanded...)
+	}
+
+	// merge files from files-from into normal args so we can reuse the normal
+	// args checks and have the ability to use both files-from and args at the
+	// same time
+	args = append(args, lines...)
+	if len(args) == 0 && !opts.Stdin {
+		return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
+	}
+
+	targets = args
+	targets, err = filterExisting(targets)
+	if err != nil {
+		return nil, err
+	}
+
+	return targets, nil
+}
+
+// parent returns the ID of the parent snapshot. If there is none, nil is
+// returned.
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string) (parentID *restic.ID, err error) {
+	// Force using a parent
+	if !opts.Force && opts.Parent != "" {
+		id, err := restic.FindSnapshot(repo, opts.Parent)
+		if err != nil {
+			return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
+		}
+
+		parentID = &id
+	}
+
+	// Find last snapshot to set it as parent, if not already set
+	if !opts.Force && parentID == nil {
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Host)
+		if err == nil {
+			parentID = &id
+		} else if err != restic.ErrNoSnapshotFound {
+			return nil, err
+		}
+	}
+
+	return parentID, nil
+}
+
+func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
+	err := opts.Check(gopts, args)
+	if err != nil {
+		return err
+	}
+
+	targets, err := collectTargets(opts, args)
+	if err != nil {
+		return err
+	}
+
+	timeStamp := time.Now()
+	if opts.TimeStamp != "" {
+		timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
+		if err != nil {
+			return errors.Fatalf("error in time option: %v\n", err)
+		}
+	}
+
+	var t tomb.Tomb
+
+	p := ui.NewBackup(term, gopts.verbosity)
+
+	// use the terminal for stdout/stderr
+	prevStdout, prevStderr := gopts.stdout, gopts.stderr
+	defer func() {
+		gopts.stdout, gopts.stderr = prevStdout, prevStderr
+	}()
+	gopts.stdout, gopts.stderr = p.Stdout(), p.Stderr()
+
+	if s, ok := os.LookupEnv("RESTIC_PROGRESS_FPS"); ok {
+		fps, err := strconv.Atoi(s)
+		if err == nil && fps >= 1 {
+			if fps > 60 {
+				fps = 60
+			}
+			p.MinUpdatePause = time.Second / time.Duration(fps)
+		}
+	}
+
+	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
+
+	p.V("open repository")
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	p.V("lock repository")
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
+	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, targets)
+	if err != nil {
+		return err
+	}
+
+	// rejectFuncs collect functions that can reject items from the backup based on path and file info
+	rejectFuncs, err := collectRejectFuncs(opts, repo, targets)
+	if err != nil {
+		return err
+	}
+
+	p.V("load index files")
+	err = repo.LoadIndex(gopts.ctx)
+	if err != nil {
+		return err
+	}
+
+	parentSnapshotID, err := findParentSnapshot(gopts.ctx, repo, opts, targets)
+	if err != nil {
+		return err
+	}
+
+	if parentSnapshotID != nil {
+		p.V("using parent snapshot %v\n", parentSnapshotID.Str())
+	}
+
+	selectByNameFilter := func(item string) bool {
+		for _, reject := range rejectByNameFuncs {
+			if reject(item) {
+				return false
+			}
+		}
+		return true
+	}
+
+	selectFilter := func(item string, fi os.FileInfo) bool {
+		for _, reject := range rejectFuncs {
+			if reject(item, fi) {
+				return false
+			}
+		}
+		return true
+	}
+
+	var targetFS fs.FS = fs.Local{}
+	if opts.Stdin {
+		p.V("read data from stdin")
+		targetFS = &fs.Reader{
+			ModTime:    timeStamp,
+			Name:       opts.StdinFilename,
+			Mode:       0644,
+			ReadCloser: os.Stdin,
+		}
+		targets = []string{opts.StdinFilename}
+	}
+
+	sc := archiver.NewScanner(targetFS)
+	sc.SelectByName = selectByNameFilter
+	sc.Select = selectFilter
+	sc.Error = p.ScannerError
+	sc.Result = p.ReportTotal
+
+	p.V("start scan on %v", targets)
+	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
+
+	arch := archiver.New(repo, targetFS, archiver.Options{})
+	arch.SelectByName = selectByNameFilter
+	arch.Select = selectFilter
+	arch.WithAtime = opts.WithAtime
+	arch.Error = p.Error
+	arch.CompleteItem = p.CompleteItemFn
+	arch.StartFile = p.StartFile
+	arch.CompleteBlob = p.CompleteBlob
+
+	if parentSnapshotID == nil {
+		parentSnapshotID = &restic.ID{}
+	}
+
+	snapshotOpts := archiver.SnapshotOptions{
+		Excludes:       opts.Excludes,
+		Tags:           opts.Tags,
+		Time:           timeStamp,
+		Hostname:       opts.Host,
+		ParentSnapshot: *parentSnapshotID,
+	}
+
+	uploader := archiver.IndexUploader{
+		Repository: repo,
+		Start: func() {
+			p.VV("uploading intermediate index")
+		},
+		Complete: func(id restic.ID) {
+			p.V("uploaded intermediate index %v", id.Str())
+		},
+	}
+
+	t.Go(func() error {
+		return uploader.Upload(gopts.ctx, t.Context(gopts.ctx), 30*time.Second)
+	})
+
+	p.V("start backup on %v", targets)
+	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
+	if err != nil {
+		return errors.Fatalf("unable to save snapshot: %v", err)
+	}
+
+	p.Finish()
+	p.P("snapshot %s saved\n", id.Str())
+
+	// cleanly shutdown all running goroutines
+	t.Kill(nil)
+
+	// let's see if one returned an error
+	err = t.Wait()
+	if err != nil {
+		return err
+	}
+
+	return nil
 }

cmd/restic/cmd_cache.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
+	"github.com/restic/restic/internal/cache"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/ui/table"
+	"github.com/spf13/cobra"
+)
+
+var cmdCache = &cobra.Command{
+	Use:   "cache",
+	Short: "Operate on local cache directories",
+	Long: `
+The "cache" command allows listing and cleaning local cache directories.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runCache(cacheOptions, globalOptions, args)
+	},
+}
+
+// CacheOptions bundles all options for the snapshots command.
+type CacheOptions struct {
+	Cleanup bool
+	MaxAge  uint
+	NoSize  bool
+}
+
+var cacheOptions CacheOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdCache)
+
+	f := cmdCache.Flags()
+	f.BoolVar(&cacheOptions.Cleanup, "cleanup", false, "remove old cache directories")
+	f.UintVar(&cacheOptions.MaxAge, "max-age", 30, "max age in `days` for cache directories to be considered old")
+	f.BoolVar(&cacheOptions.NoSize, "no-size", false, "do not output the size of the cache directories")
+}
+
+func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
+	if len(args) > 0 {
+		return errors.Fatal("the cache command has no arguments")
+	}
+
+	if gopts.NoCache {
+		return errors.Fatal("Refusing to do anything, the cache is disabled")
+	}
+
+	var (
+		cachedir = gopts.CacheDir
+		err      error
+	)
+
+	if cachedir == "" {
+		cachedir, err = cache.DefaultDir()
+		if err != nil {
+			return err
+		}
+	}
+
+	if opts.Cleanup || gopts.CleanupCache {
+		oldDirs, err := cache.OlderThan(cachedir, time.Duration(opts.MaxAge)*24*time.Hour)
+		if err != nil {
+			return err
+		}
+
+		if len(oldDirs) == 0 {
+			Verbosef("no old cache dirs found\n")
+			return nil
+		}
+
+		Verbosef("remove %d old cache directories\n", len(oldDirs))
+
+		for _, item := range oldDirs {
+			dir := filepath.Join(cachedir, item.Name())
+			err = fs.RemoveAll(dir)
+			if err != nil {
+				Warnf("unable to remove %v: %v\n", dir, err)
+			}
+		}
+
+		return nil
+	}
+
+	tab := table.New()
+
+	type data struct {
+		ID   string
+		Last string
+		Old  string
+		Size string
+	}
+
+	tab.AddColumn("Repo ID", "{{ .ID }}")
+	tab.AddColumn("Last Used", "{{ .Last }}")
+	tab.AddColumn("Old", "{{ .Old }}")
+
+	if !opts.NoSize {
+		tab.AddColumn("Size", "{{ .Size }}")
+	}
+
+	dirs, err := cache.All(cachedir)
+	if err != nil {
+		return err
+	}
+
+	if len(dirs) == 0 {
+		Printf("no cache dirs found, basedir is %v\n", cachedir)
+		return nil
+	}
+
+	sort.Slice(dirs, func(i, j int) bool {
+		return dirs[i].ModTime().Before(dirs[j].ModTime())
+	})
+
+	for _, entry := range dirs {
+		var old string
+		if cache.IsOld(entry.ModTime(), time.Duration(opts.MaxAge)*24*time.Hour) {
+			old = "yes"
+		}
+
+		var size string
+		if !opts.NoSize {
+			bytes, err := dirSize(filepath.Join(cachedir, entry.Name()))
+			if err != nil {
+				return err
+			}
+			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
+		}
+
+		tab.AddRow(data{
+			entry.Name()[:10],
+			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
+			old,
+			size,
+		})
+	}
+
+	tab.Write(gopts.stdout)
+	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
+
+	return nil
+}
+
+func dirSize(path string) (int64, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if err != nil || info == nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			size += info.Size()
+		}
+
+		return nil
+	})
+	return size, err
+}

cmd/restic/cmd_cat.go

@@ -58,7 +58,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			// find snapshot id with prefix
 			id, err = restic.FindSnapshot(repo, args[1])
 			if err != nil {
-				return err
+				return errors.Fatalf("could not find snapshot: %v\n", err)
 			}
 		}
 	}

cmd/restic/cmd_check.go

@@ -2,13 +2,17 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 )
 
@@ -26,13 +30,17 @@ repository and not use a local cache.
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCheck(checkOptions, globalOptions, args)
 	},
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		return checkFlags(checkOptions)
+	},
 }
 
 // CheckOptions bundles all options for the 'check' command.
 type CheckOptions struct {
-	ReadData    bool
-	CheckUnused bool
-	WithCache   bool
+	ReadData       bool
+	ReadDataSubset string
+	CheckUnused    bool
+	WithCache      bool
 }
 
 var checkOptions CheckOptions
@@ -42,10 +50,45 @@ func init() {
 
 	f := cmdCheck.Flags()
 	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
+	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read subset n of m data packs (format: `n/m`)")
 	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
 }
 
+func checkFlags(opts CheckOptions) error {
+	if opts.ReadData && opts.ReadDataSubset != "" {
+		return errors.Fatalf("check flags --read-data and --read-data-subset cannot be used together")
+	}
+	if opts.ReadDataSubset != "" {
+		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
+		if err != nil || len(dataSubset) != 2 {
+			return errors.Fatalf("check flag --read-data-subset must have two positive integer values, e.g. --read-data-subset=1/2")
+		}
+		if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
+			return errors.Fatalf("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
+		}
+	}
+
+	return nil
+}
+
+// stringToIntSlice converts string to []uint, using '/' as element separator
+func stringToIntSlice(param string) (split []uint, err error) {
+	if param == "" {
+		return nil, nil
+	}
+	parts := strings.Split(param, "/")
+	result := make([]uint, len(parts))
+	for idx, part := range parts {
+		uintval, err := strconv.ParseUint(part, 10, 0)
+		if err != nil {
+			return nil, err
+		}
+		result[idx] = uint(uintval)
+	}
+	return result, nil
+}
+
 func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
 	if gopts.Quiet {
 		return nil
@@ -76,15 +119,58 @@ func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
 	return readProgress
 }
 
+// prepareCheckCache configures a special cache directory for check.
+//
+//  * if --with-cache is specified, the default cache is used
+//  * if the user explicitly requested --no-cache, we don't use any cache
+//  * if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
+//  * by default, we use a cache in a temporary directory that is deleted after the check
+func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
+	cleanup = func() {}
+	if opts.WithCache {
+		// use the default cache, no setup needed
+		return cleanup
+	}
+
+	if gopts.NoCache {
+		// don't use any cache, no setup needed
+		return cleanup
+	}
+
+	cachedir := gopts.CacheDir
+
+	// use a cache in a temporary directory
+	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
+	if err != nil {
+		// if an error occurs, don't use any cache
+		Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
+		gopts.NoCache = true
+		return cleanup
+	}
+
+	gopts.CacheDir = tempdir
+	Verbosef("using temporary cache in %v\n", tempdir)
+
+	cleanup = func() {
+		err := fs.RemoveAll(tempdir)
+		if err != nil {
+			Warnf("error removing temporary cache directory: %v\n", err)
+		}
+	}
+
+	return cleanup
+}
+
 func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
 		return errors.Fatal("check has no arguments")
 	}
 
-	if !opts.WithCache {
-		// do not use a cache for the checker
-		gopts.NoCache = true
-	}
+	cleanup := prepareCheckCache(opts, &gopts)
+	AddCleanupHandler(func() error {
+		cleanup()
+		return nil
+	})
 
 	repo, err := OpenRepository(gopts)
 	if err != nil {
@@ -114,7 +200,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if dupFound {
-		Printf("\nrun `restic rebuild-index' to correct this\n")
+		Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
 	}
 
 	if len(errs) > 0 {
@@ -125,16 +211,26 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	errorsFound := false
+	orphanedPacks := 0
 	errChan := make(chan error)
 
 	Verbosef("check all packs\n")
 	go chkr.Packs(gopts.ctx, errChan)
 
 	for err := range errChan {
+		if checker.IsOrphanedPack(err) {
+			orphanedPacks++
+			Verbosef("%v\n", err)
+			continue
+		}
 		errorsFound = true
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 	}
 
+	if orphanedPacks > 0 {
+		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nYou can run `restic prune` to correct this.\n", orphanedPacks)
+	}
+
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
 	go chkr.Structure(gopts.ctx, errChan)
@@ -158,13 +254,25 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	if opts.ReadData {
-		Verbosef("read all data\n")
+	doReadData := func(bucket, totalBuckets uint) {
+		packs := restic.IDSet{}
+		for pack := range chkr.GetPacks() {
+			if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
+				packs.Insert(pack)
+			}
+		}
+		packCount := uint64(len(packs))
 
-		p := newReadProgress(gopts, restic.Stat{Blobs: chkr.CountPacks()})
+		if packCount < chkr.CountPacks() {
+			Verbosef(fmt.Sprintf("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets))
+		} else {
+			Verbosef("read all data\n")
+		}
+
+		p := newReadProgress(gopts, restic.Stat{Blobs: packCount})
 		errChan := make(chan error)
 
-		go chkr.ReadData(gopts.ctx, p, errChan)
+		go chkr.ReadPacks(gopts.ctx, packs, p, errChan)
 
 		for err := range errChan {
 			errorsFound = true
@@ -172,6 +280,14 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	switch {
+	case opts.ReadData:
+		doReadData(1, 1)
+	case opts.ReadDataSubset != "":
+		dataSubset, _ := stringToIntSlice(opts.ReadDataSubset)
+		doReadData(dataSubset[0], dataSubset[1])
+	}
+
 	if errorsFound {
 		return errors.Fatal("repository contains errors")
 	}

cmd/restic/cmd_diff.go

@@ -21,11 +21,11 @@ The "diff" command shows differences from the first to the second snapshot. The
 first characters in each line display what has happened to a particular file or
 directory:
 
- +  The item was added
- -  The item was removed
- U  The metadata (access mode, timestamps, ...) for the item was updated
- M  The file's content was modified
- T  The type was changed, e.g. a file was made a symlink
+* +  The item was added
+* -  The item was removed
+* U  The metadata (access mode, timestamps, ...) for the item was updated
+* M  The file's content was modified
+* T  The type was changed, e.g. a file was made a symlink
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/restic/cmd_find.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -11,31 +10,44 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
 )
 
 var cmdFind = &cobra.Command{
-	Use:   "find [flags] PATTERN",
-	Short: "Find a file or directory",
+	Use:   "find [flags] PATTERN...",
+	Short: "Find a file, a directory or restic IDs",
 	Long: `
 The "find" command searches for files or directories in snapshots stored in the
-repo. `,
+repo.
+It can also be used to search for restic blobs or trees for troubleshooting.`,
+	Example: `restic find config.json
+restic find --json "*.yml" "*.json"
+restic find --json --blob 420f620f b46ebe8a ddd38656
+restic find --show-pack-id --blob 420f620f
+restic find --tree 577c2bc9 f81f2e22 a62827a9
+restic find --pack 025c1d06`,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runFind(findOptions, globalOptions, args)
 	},
 }
 
+const shortStr = 8 // Length of short IDs: 4 bytes as hex strings
+
 // FindOptions bundles all options for the find command.
 type FindOptions struct {
-	Oldest          string
-	Newest          string
-	Snapshots       []string
-	CaseInsensitive bool
-	ListLong        bool
-	Host            string
-	Paths           []string
-	Tags            restic.TagLists
+	Oldest             string
+	Newest             string
+	Snapshots          []string
+	BlobID, TreeID     bool
+	PackID, ShowPackID bool
+	CaseInsensitive    bool
+	ListLong           bool
+	Host               string
+	Paths              []string
+	Tags               restic.TagLists
 }
 
 var findOptions FindOptions
@@ -47,6 +59,10 @@ func init() {
 	f.StringVarP(&findOptions.Oldest, "oldest", "O", "", "oldest modification date/time")
 	f.StringVarP(&findOptions.Newest, "newest", "N", "", "newest modification date/time")
 	f.StringArrayVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.BoolVar(&findOptions.BlobID, "blob", false, "pattern is a blob-ID")
+	f.BoolVar(&findOptions.TreeID, "tree", false, "pattern is a tree-ID")
+	f.BoolVar(&findOptions.PackID, "pack", false, "pattern is a pack-ID")
+	f.BoolVar(&findOptions.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob)")
 	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
 	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 
@@ -57,7 +73,7 @@ func init() {
 
 type findPattern struct {
 	oldest, newest time.Time
-	pattern        string
+	pattern        []string
 	ignoreCase     bool
 }
 
@@ -94,7 +110,7 @@ type statefulOutput struct {
 	hits     int
 }
 
-func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
+func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
 	type findNode restic.Node
 	b, err := json.Marshal(struct {
 		// Add these attributes
@@ -111,7 +127,7 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
 		Content            byte `json:"content,omitempty"`
 		Subtree            byte `json:"subtree,omitempty"`
 	}{
-		Path:        filepath.Join(prefix, node.Name),
+		Path:        path,
 		Permissions: node.Mode.String(),
 		findNode:    (*findNode)(node),
 	})
@@ -138,22 +154,73 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
 	s.hits++
 }
 
-func (s *statefulOutput) PrintNormal(prefix string, node *restic.Node) {
+func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) {
 	if s.newsn != s.oldsn {
 		if s.oldsn != nil {
 			Verbosef("\n")
 		}
 		s.oldsn = s.newsn
-		Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID())
+		Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID().Str())
 	}
-	Printf(formatNode(prefix, node, s.ListLong) + "\n")
+	Printf(formatNode(path, node, s.ListLong) + "\n")
 }
 
-func (s *statefulOutput) Print(prefix string, node *restic.Node) {
+func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
 	if s.JSON {
-		s.PrintJSON(prefix, node)
+		s.PrintPatternJSON(path, node)
 	} else {
-		s.PrintNormal(prefix, node)
+		s.PrintPatternNormal(path, node)
+	}
+}
+
+func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+	b, err := json.Marshal(struct {
+		// Add these attributes
+		ObjectType string    `json:"object_type"`
+		ID         string    `json:"id"`
+		Path       string    `json:"path"`
+		ParentTree string    `json:"parent_tree,omitempty"`
+		SnapshotID string    `json:"snapshot"`
+		Time       time.Time `json:"time,omitempty"`
+	}{
+		ObjectType: kind,
+		ID:         id,
+		Path:       nodepath,
+		SnapshotID: sn.ID().String(),
+		ParentTree: treeID,
+		Time:       sn.Time,
+	})
+	if err != nil {
+		Warnf("Marshall failed: %v\n", err)
+		return
+	}
+	if !s.inuse {
+		Printf("[")
+		s.inuse = true
+	}
+	if s.hits > 0 {
+		Printf(",")
+	}
+	Printf(string(b))
+	s.hits++
+}
+
+func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+	Printf("Found %s %s\n", kind, id)
+	if kind == "blob" {
+		Printf(" ... in file %s\n", nodepath)
+		Printf("     (tree %s)\n", treeID)
+	} else {
+		Printf(" ... path %s\n", nodepath)
+	}
+	Printf(" ... in snapshot %s (%s)\n", sn.ID().Str(), sn.Time.Format(TimeFormat))
+}
+
+func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
+	if s.JSON {
+		s.PrintObjectJSON(kind, id, nodepath, treeID, sn)
+	} else {
+		s.PrintObjectNormal(kind, id, nodepath, treeID, sn)
 	}
 }
 
@@ -174,85 +241,242 @@ func (s *statefulOutput) Finish() {
 
 // Finder bundles information needed to find a file or directory.
 type Finder struct {
-	repo     restic.Repository
-	pat      findPattern
-	out      statefulOutput
-	notfound restic.IDSet
-}
-
-func (f *Finder) findInTree(ctx context.Context, treeID restic.ID, prefix string) error {
-	if f.notfound.Has(treeID) {
-		debug.Log("%v skipping tree %v, has already been checked", prefix, treeID)
-		return nil
-	}
-
-	debug.Log("%v checking tree %v\n", prefix, treeID)
-
-	tree, err := f.repo.LoadTree(ctx, treeID)
-	if err != nil {
-		return err
-	}
-
-	var found bool
-	for _, node := range tree.Nodes {
-		debug.Log("  testing entry %q\n", node.Name)
-
-		name := node.Name
-		if f.pat.ignoreCase {
-			name = strings.ToLower(name)
-		}
-
-		m, err := filepath.Match(f.pat.pattern, name)
-		if err != nil {
-			return err
-		}
-
-		if m {
-			if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
-				debug.Log("    ModTime is older than %s\n", f.pat.oldest)
-				continue
-			}
-
-			if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
-				debug.Log("    ModTime is newer than %s\n", f.pat.newest)
-				continue
-			}
-
-			debug.Log("    found match\n")
-			found = true
-			f.out.Print(prefix, node)
-		}
-
-		if node.Type == "dir" {
-			if err := f.findInTree(ctx, *node.Subtree, filepath.Join(prefix, node.Name)); err != nil {
-				return err
-			}
-		}
-	}
-
-	if !found {
-		f.notfound.Insert(treeID)
-	}
-
-	return nil
+	repo        restic.Repository
+	pat         findPattern
+	out         statefulOutput
+	ignoreTrees restic.IDSet
+	blobIDs     map[string]struct{}
+	treeIDs     map[string]struct{}
+	itemsFound  int
 }
 
 func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
 	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
 
+	if sn.Tree == nil {
+		return errors.Errorf("snapshot %v has no tree", sn.ID().Str())
+	}
+
 	f.out.newsn = sn
-	return f.findInTree(ctx, *sn.Tree, string(filepath.Separator))
+	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+		if err != nil {
+			return false, err
+		}
+
+		if node == nil {
+			return false, nil
+		}
+
+		normalizedNodepath := nodepath
+		if f.pat.ignoreCase {
+			normalizedNodepath = strings.ToLower(nodepath)
+		}
+
+		var foundMatch bool
+
+		for _, pat := range f.pat.pattern {
+			found, err := filter.Match(pat, normalizedNodepath)
+			if err != nil {
+				return false, err
+			}
+			if found {
+				foundMatch = true
+				break
+			}
+		}
+
+		var (
+			ignoreIfNoMatch = true
+			errIfNoMatch    error
+		)
+		if node.Type == "dir" {
+			var childMayMatch bool
+			for _, pat := range f.pat.pattern {
+				mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
+				if err != nil {
+					return false, err
+				}
+				if mayMatch {
+					childMayMatch = true
+					break
+				}
+			}
+
+			if !childMayMatch {
+				ignoreIfNoMatch = true
+				errIfNoMatch = walker.SkipNode
+			} else {
+				ignoreIfNoMatch = false
+			}
+		}
+
+		if !foundMatch {
+			return ignoreIfNoMatch, errIfNoMatch
+		}
+
+		if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
+			debug.Log("    ModTime is older than %s\n", f.pat.oldest)
+			return ignoreIfNoMatch, errIfNoMatch
+		}
+
+		if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
+			debug.Log("    ModTime is newer than %s\n", f.pat.newest)
+			return ignoreIfNoMatch, errIfNoMatch
+		}
+
+		debug.Log("    found match\n")
+		f.out.PrintPattern(nodepath, node)
+		return false, nil
+	})
+}
+
+func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
+	debug.Log("searching IDs in snapshot %s", sn.ID())
+
+	if sn.Tree == nil {
+		return errors.Errorf("snapshot %v has no tree", sn.ID().Str())
+	}
+
+	f.out.newsn = sn
+	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+		if err != nil {
+			return false, err
+		}
+
+		if node == nil {
+			return false, nil
+		}
+
+		if node.Type == "dir" && f.treeIDs != nil {
+			treeID := node.Subtree
+			found := false
+			if _, ok := f.treeIDs[treeID.Str()]; ok {
+				found = true
+			} else if _, ok := f.treeIDs[treeID.String()]; ok {
+				found = true
+			}
+			if found {
+				f.out.PrintObject("tree", treeID.String(), nodepath, "", sn)
+				f.itemsFound++
+				// Terminate if we have found all trees (and we are not
+				// looking for blobs)
+				if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
+					// Return an error to terminate the Walk
+					return true, errors.New("OK")
+				}
+			}
+		}
+
+		if node.Type == "file" && f.blobIDs != nil {
+			for _, id := range node.Content {
+				idStr := id.String()
+				if _, ok := f.blobIDs[idStr]; !ok {
+					// Look for short ID form
+					if _, ok := f.blobIDs[idStr[:shortStr]]; !ok {
+						continue
+					}
+					// Replace the short ID with the long one
+					f.blobIDs[idStr] = struct{}{}
+					delete(f.blobIDs, idStr[:shortStr])
+				}
+				f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
+				break
+			}
+		}
+
+		return false, nil
+	})
+}
+
+// packsToBlobs converts the list of pack IDs to a list of blob IDs that
+// belong to those packs.
+func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
+	packIDs := make(map[string]struct{})
+	for _, p := range packs {
+		packIDs[p] = struct{}{}
+	}
+	if f.blobIDs == nil {
+		f.blobIDs = make(map[string]struct{})
+	}
+
+	allPacksFound := false
+	packsFound := 0
+
+	debug.Log("Looking for packs...")
+	err := f.repo.List(ctx, restic.DataFile, func(id restic.ID, size int64) error {
+		if allPacksFound {
+			return nil
+		}
+		idStr := id.String()
+		if _, ok := packIDs[idStr]; !ok {
+			// Look for short ID form
+			if _, ok := packIDs[idStr[:shortStr]]; !ok {
+				return nil
+			}
+		}
+		debug.Log("Found pack %s", idStr)
+		blobs, _, err := f.repo.ListPack(ctx, id, size)
+		if err != nil {
+			return err
+		}
+		for _, b := range blobs {
+			f.blobIDs[b.ID.String()] = struct{}{}
+		}
+		// Stop searching when all packs have been found
+		packsFound++
+		if packsFound >= len(packIDs) {
+			allPacksFound = true
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if !allPacksFound {
+		return errors.Fatal("unable to find all specified pack(s)")
+	}
+
+	debug.Log("%d blobs found", len(f.blobIDs))
+	return nil
+}
+
+func (f *Finder) findBlobsPacks(ctx context.Context) {
+	idx := f.repo.Index()
+	for i := range f.blobIDs {
+		rid, err := restic.ParseID(i)
+		if err != nil {
+			Printf("Note: cannot find pack for blob '%s', unable to parse ID: %v\n", i, err)
+			continue
+		}
+
+		blobs, found := idx.Lookup(rid, restic.DataBlob)
+		if !found {
+			Printf("Blob %s not found in the index\n", rid.Str())
+			continue
+		}
+
+		for _, b := range blobs {
+			if b.ID.Equal(rid) {
+				Printf("Blob belongs to pack %s\n ... Pack %s: %s\n", b.PackID, b.PackID.Str(), b.String())
+				break
+			}
+		}
+	}
 }
 
 func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
-	if len(args) != 1 {
+	if len(args) == 0 {
 		return errors.Fatal("wrong number of arguments")
 	}
 
 	var err error
-	pat := findPattern{pattern: args[0]}
+	pat := findPattern{pattern: args}
 	if opts.CaseInsensitive {
-		pat.pattern = strings.ToLower(pat.pattern)
+		for i := range pat.pattern {
+			pat.pattern[i] = strings.ToLower(pat.pattern[i])
+		}
 		pat.ignoreCase = true
 	}
 
@@ -268,6 +492,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	// Check at most only one kind of IDs is provided: currently we
+	// can't mix types
+	if (opts.BlobID && opts.TreeID) ||
+		(opts.BlobID && opts.PackID) ||
+		(opts.TreeID && opts.PackID) {
+		return errors.Fatal("cannot have several ID types")
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -289,17 +521,45 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	defer cancel()
 
 	f := &Finder{
-		repo:     repo,
-		pat:      pat,
-		out:      statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
-		notfound: restic.NewIDSet(),
+		repo:        repo,
+		pat:         pat,
+		out:         statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
+		ignoreTrees: restic.NewIDSet(),
 	}
+
+	if opts.BlobID {
+		f.blobIDs = make(map[string]struct{})
+		for _, pat := range f.pat.pattern {
+			f.blobIDs[pat] = struct{}{}
+		}
+	}
+	if opts.TreeID {
+		f.treeIDs = make(map[string]struct{})
+		for _, pat := range f.pat.pattern {
+			f.treeIDs[pat] = struct{}{}
+		}
+	}
+
+	if opts.PackID {
+		f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
+	}
+
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
+		if f.blobIDs != nil || f.treeIDs != nil {
+			if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
+				return err
+			}
+			continue
+		}
 		if err = f.findInSnapshot(ctx, sn); err != nil {
 			return err
 		}
 	}
 	f.out.Finish()
 
+	if opts.ShowPackID && f.blobIDs != nil {
+		f.findBlobsPacks(ctx)
+	}
+
 	return nil
 }

cmd/restic/cmd_forget.go

@@ -33,6 +33,7 @@ type ForgetOptions struct {
 	Weekly   int
 	Monthly  int
 	Yearly   int
+	Within   restic.Duration
 	KeepTags restic.TagLists
 
 	Host    string
@@ -58,13 +59,15 @@ func init() {
 	f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots")
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
+	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are older than `duration` (eg. 1y5m7d) relative to the latest snapshot")
 
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
-	// Sadly the commonly used shortcut `H` is already used.
 	f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
-	// Deprecated since 2017-03-07.
-	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
+	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname`")
+	f.MarkDeprecated("hostname", "use --host")
+
 	f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
+
 	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 	f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact format")
 
@@ -170,6 +173,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		Weekly:  opts.Weekly,
 		Monthly: opts.Monthly,
 		Yearly:  opts.Yearly,
+		Within:  opts.Within,
 		Tags:    opts.KeepTags,
 	}
 
@@ -178,6 +182,8 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if !policy.Empty() {
+		Verbosef("Applying Policy: %v\n", policy)
+
 		for k, snapshotGroup := range snapshotGroups {
 			var key key
 			if json.Unmarshal([]byte(k), &key) != nil {
@@ -201,17 +207,17 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 			}
 			Verbosef(":\n\n")
 
-			keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
+			keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
 
 			if len(keep) != 0 && !gopts.Quiet {
 				Printf("keep %d snapshots:\n", len(keep))
-				PrintSnapshots(globalOptions.stdout, keep, opts.Compact)
+				PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
 				Printf("\n")
 			}
 
 			if len(remove) != 0 && !gopts.Quiet {
 				Printf("remove %d snapshots:\n", len(remove))
-				PrintSnapshots(globalOptions.stdout, remove, opts.Compact)
+				PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
 				Printf("\n")
 			}
 

cmd/restic/cmd_key.go

@@ -2,11 +2,15 @@ package main
 
 import (
 	"context"
-	"fmt"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"strings"
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/table"
 
 	"github.com/spf13/cobra"
 )
@@ -23,14 +27,25 @@ The "key" command manages keys (passwords) for accessing the repository.
 	},
 }
 
+var newPasswordFile string
+
 func init() {
 	cmdRoot.AddCommand(cmdKey)
+
+	flags := cmdKey.Flags()
+	flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "the file from which to load a new password")
 }
 
-func listKeys(ctx context.Context, s *repository.Repository) error {
-	tab := NewTable()
-	tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
-	tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
+func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
+	type keyInfo struct {
+		Current  bool   `json:"current"`
+		ID       string `json:"id"`
+		UserName string `json:"userName"`
+		HostName string `json:"hostName"`
+		Created  string `json:"created"`
+	}
+
+	var keys []keyInfo
 
 	err := s.List(ctx, restic.KeyFile, func(id restic.ID, size int64) error {
 		k, err := repository.LoadKey(ctx, s, id.String())
@@ -39,20 +54,36 @@ func listKeys(ctx context.Context, s *repository.Repository) error {
 			return nil
 		}
 
-		var current string
-		if id.String() == s.KeyName() {
-			current = "*"
-		} else {
-			current = " "
+		key := keyInfo{
+			Current:  id.String() == s.KeyName(),
+			ID:       id.Str(),
+			UserName: k.Username,
+			HostName: k.Hostname,
+			Created:  k.Created.Format(TimeFormat),
 		}
-		tab.Rows = append(tab.Rows, []interface{}{current, id.Str(),
-			k.Username, k.Hostname, k.Created.Format(TimeFormat)})
+
+		keys = append(keys, key)
 		return nil
 	})
+
 	if err != nil {
 		return err
 	}
 
+	if gopts.JSON {
+		return json.NewEncoder(globalOptions.stdout).Encode(keys)
+	}
+
+	tab := table.New()
+	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
+	tab.AddColumn("User", "{{ .UserName }}")
+	tab.AddColumn("Host", "{{ .HostName }}")
+	tab.AddColumn("Created", "{{ .Created }}")
+
+	for _, key := range keys {
+		tab.AddRow(key)
+	}
+
 	return tab.Write(globalOptions.stdout)
 }
 
@@ -64,6 +95,10 @@ func getNewPassword(gopts GlobalOptions) (string, error) {
 		return testKeyNewPassword, nil
 	}
 
+	if newPasswordFile != "" {
+		return loadPasswordFromFile(newPasswordFile)
+	}
+
 	// Since we already have an open repository, temporary remove the password
 	// to prompt the user for the passwd.
 	newopts := gopts
@@ -148,7 +183,7 @@ func runKey(gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		return listKeys(ctx, repo)
+		return listKeys(ctx, repo, gopts)
 	case "add":
 		lock, err := lockRepo(repo)
 		defer unlockRepo(lock)
@@ -182,3 +217,11 @@ func runKey(gopts GlobalOptions, args []string) error {
 
 	return nil
 }
+
+func loadPasswordFromFile(pwdFile string) (string, error) {
+	s, err := ioutil.ReadFile(pwdFile)
+	if os.IsNotExist(err) {
+		return "", errors.Fatalf("%s does not exist", pwdFile)
+	}
+	return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
+}

cmd/restic/cmd_list.go

@@ -18,7 +18,7 @@ The "list" command allows listing objects in the repository based on type.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runList(globalOptions, args)
+		return runList(cmd, globalOptions, args)
 	},
 }
 
@@ -26,9 +26,9 @@ func init() {
 	cmdRoot.AddCommand(cmdList)
 }
 
-func runList(opts GlobalOptions, args []string) error {
+func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatal("type not specified")
+		return errors.Fatal("type not specified, usage: " + cmd.Use)
 	}
 
 	repo, err := OpenRepository(opts)

cmd/restic/cmd_ls.go

@@ -2,22 +2,37 @@ package main
 
 import (
 	"context"
-	"path/filepath"
+	"encoding/json"
+	"os"
+	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
 )
 
 var cmdLs = &cobra.Command{
-	Use:   "ls [flags] [snapshot-ID ...]",
+	Use:   "ls [flags] [snapshotID] [dir...]",
 	Short: "List files in a snapshot",
 	Long: `
-The "ls" command allows listing files and directories in a snapshot.
+The "ls" command lists files and directories in a snapshot.
 
-The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
+The special snapshot ID "latest" can be used to list files and
+directories of the latest snapshot in the repository. The
+--host flag can be used in conjunction to select the latest
+snapshot originating from a certain host only.
+
+File listings can optionally be filtered by directories. Any
+positional arguments after the snapshot ID are interpreted as
+absolute directory paths, and only files inside those directories
+will be listed. If the --recursive flag is used, then the filter
+will allow traversing into matching directories' subfolders.
+Any directory paths specified must be absolute (starting with
+a path separator); paths use the forward slash '/' as separator.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -27,10 +42,11 @@ The special snapshot-ID "latest" can be used to list files and directories of th
 
 // LsOptions collects all options for the ls command.
 type LsOptions struct {
-	ListLong bool
-	Host     string
-	Tags     restic.TagLists
-	Paths    []string
+	ListLong  bool
+	Host      string
+	Tags      restic.TagLists
+	Paths     []string
+	Recursive bool
 }
 
 var lsOptions LsOptions
@@ -40,29 +56,33 @@ func init() {
 
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-
 	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
 	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
 	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 }
 
-func printTree(ctx context.Context, repo *repository.Repository, id *restic.ID, prefix string) error {
-	tree, err := repo.LoadTree(ctx, *id)
-	if err != nil {
-		return err
-	}
+type lsSnapshot struct {
+	*restic.Snapshot
 
-	for _, entry := range tree.Nodes {
-		Printf("%s\n", formatNode(prefix, entry, lsOptions.ListLong))
+	ID         *restic.ID `json:"id"`
+	ShortID    string     `json:"short_id"`
+	Nodes      []lsNode   `json:"nodes"`
+	StructType string     `json:"struct_type"` // "snapshot"
+}
 
-		if entry.Type == "dir" && entry.Subtree != nil {
-			if err = printTree(ctx, repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
+type lsNode struct {
+	Name       string      `json:"name"`
+	Type       string      `json:"type"`
+	Path       string      `json:"path"`
+	UID        uint32      `json:"uid"`
+	GID        uint32      `json:"gid"`
+	Size       uint64      `json:"size,omitempty"`
+	Mode       os.FileMode `json:"mode,omitempty"`
+	ModTime    time.Time   `json:"mtime,omitempty"`
+	AccessTime time.Time   `json:"atime,omitempty"`
+	ChangeTime time.Time   `json:"ctime,omitempty"`
+	StructType string      `json:"struct_type"` // "node"
 }
 
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
@@ -70,6 +90,51 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
 	}
 
+	// extract any specific directories to walk
+	var dirs []string
+	if len(args) > 1 {
+		dirs = args[1:]
+		for _, dir := range dirs {
+			if !strings.HasPrefix(dir, "/") {
+				return errors.Fatal("All path filters must be absolute, starting with a forward slash '/'")
+			}
+		}
+	}
+
+	withinDir := func(nodepath string) bool {
+		if len(dirs) == 0 {
+			return true
+		}
+
+		for _, dir := range dirs {
+			// we're within one of the selected dirs, example:
+			//   nodepath: "/test/foo"
+			//   dir:      "/test"
+			if fs.HasPathPrefix(dir, nodepath) {
+				return true
+			}
+		}
+		return false
+	}
+
+	approachingMatchingTree := func(nodepath string) bool {
+		if len(dirs) == 0 {
+			return true
+		}
+
+		for _, dir := range dirs {
+			// the current node path is a prefix for one of the
+			// directories, so we're interested in something deeper in the
+			// tree. Example:
+			//   nodepath: "/test"
+			//   dir:      "/test/foo"
+			if fs.HasPathPrefix(nodepath, dir) {
+				return true
+			}
+		}
+		return false
+	}
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -81,12 +146,99 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
-		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
 
-		if err = printTree(gopts.ctx, repo, sn.Tree, string(filepath.Separator)); err != nil {
+	var (
+		printSnapshot func(sn *restic.Snapshot)
+		printNode     func(path string, node *restic.Node)
+		printFinish   func() error
+	)
+
+	if gopts.JSON {
+		var lssnapshots []lsSnapshot
+
+		printSnapshot = func(sn *restic.Snapshot) {
+			lss := lsSnapshot{
+				Snapshot:   sn,
+				ID:         sn.ID(),
+				ShortID:    sn.ID().Str(),
+				StructType: "snapshot",
+			}
+			lssnapshots = append(lssnapshots, lss)
+		}
+
+		printNode = func(path string, node *restic.Node) {
+			lsn := lsNode{
+				Name:       node.Name,
+				Type:       node.Type,
+				Path:       path,
+				UID:        node.UID,
+				GID:        node.GID,
+				Size:       node.Size,
+				Mode:       node.Mode,
+				ModTime:    node.ModTime,
+				AccessTime: node.AccessTime,
+				ChangeTime: node.ChangeTime,
+				StructType: "node",
+			}
+			s := &lssnapshots[len(lssnapshots)-1]
+			s.Nodes = append(s.Nodes, lsn)
+		}
+
+		printFinish = func() error {
+			return json.NewEncoder(gopts.stdout).Encode(lssnapshots)
+		}
+	} else {
+		// default output methods
+		printSnapshot = func(sn *restic.Snapshot) {
+			Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
+		}
+		printNode = func(path string, node *restic.Node) {
+			Printf("%s\n", formatNode(path, node, lsOptions.ListLong))
+		}
+		printFinish = func() error {
+			return nil
+		}
+	}
+
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) {
+		printSnapshot(sn)
+
+		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+			if err != nil {
+				return false, err
+			}
+			if node == nil {
+				return false, nil
+			}
+
+			if withinDir(nodepath) {
+				// if we're within a dir, print the node
+				printNode(nodepath, node)
+
+				// if recursive listing is requested, signal the walker that it
+				// should continue walking recursively
+				if opts.Recursive {
+					return false, nil
+				}
+			}
+
+			// if there's an upcoming match deeper in the tree (but we're not
+			// there yet), signal the walker to descend into any subdirs
+			if approachingMatchingTree(nodepath) {
+				return false, nil
+			}
+
+			// otherwise, signal the walker to not walk recursively into any
+			// subdirs
+			if node.Type == "dir" {
+				return false, walker.SkipNode
+			}
+			return false, nil
+		})
+
+		if err != nil {
 			return err
 		}
 	}
-	return nil
+	return printFinish()
 }

cmd/restic/cmd_mount.go

@@ -1,4 +1,6 @@
+// +build !netbsd
 // +build !openbsd
+// +build !solaris
 // +build !windows
 
 package main

cmd/restic/cmd_prune.go

@@ -149,8 +149,8 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
-	duplicateBlobs := 0
-	duplicateBytes := 0
+	var duplicateBlobs uint64
+	var duplicateBytes uint64
 
 	// find duplicate blobs
 	for _, p := range idx.Packs {
@@ -161,7 +161,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 
 			if blobCount[h] > 1 {
 				duplicateBlobs++
-				duplicateBytes += int(entry.Length)
+				duplicateBytes += uint64(entry.Length)
 			}
 		}
 	}
@@ -252,7 +252,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 				continue
 			}
 
-			removeBytes += int(blob.Length)
+			removeBytes += uint64(blob.Length)
 		}
 
 		if hasActiveBlob {

cmd/restic/cmd_restore.go

@@ -5,6 +5,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/restorer"
 
 	"github.com/spf13/cobra"
 )
@@ -33,6 +34,7 @@ type RestoreOptions struct {
 	Host    string
 	Paths   []string
 	Tags    restic.TagLists
+	Verify  bool
 }
 
 var restoreOptions RestoreOptions
@@ -48,6 +50,7 @@ func init() {
 	flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
 	flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
 	flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
 }
 
 func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
@@ -104,7 +107,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	res, err := restic.NewRestorer(repo, id)
+	res, err := restorer.NewRestorer(repo, id)
 	if err != nil {
 		Exitf(2, "creating restorer failed: %v\n", err)
 	}
@@ -153,6 +156,12 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target)
 
 	err = res.RestoreTo(ctx, opts.Target)
+	if err == nil && opts.Verify {
+		Verbosef("verifying files in %s\n", opts.Target)
+		var count int
+		count, err = res.VerifyFiles(ctx, opts.Target)
+		Verbosef("finished verifying %d files in %s\n", count, opts.Target)
+	}
 	if totalErrors > 0 {
 		Printf("There were %d errors\n", totalErrors)
 	}

cmd/restic/cmd_self_update.go

@@ -0,0 +1,51 @@
+// +build selfupdate
+
+package main
+
+import (
+	"os"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/selfupdate"
+	"github.com/spf13/cobra"
+)
+
+var cmdSelfUpdate = &cobra.Command{
+	Use:   "self-update [flags]",
+	Short: "Update the restic binary",
+	Long: `
+The command "update-restic" downloads the latest stable release of restic from
+GitHub and replaces the currently running binary. After download, the
+authenticity of the binary is verified using the GPG signature on the release
+files.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runSelfUpdate(selfUpdateOptions, globalOptions, args)
+	},
+}
+
+// SelfUpdateOptions collects all options for the update-restic command.
+type SelfUpdateOptions struct {
+	Output string
+}
+
+var selfUpdateOptions SelfUpdateOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdSelfUpdate)
+
+	flags := cmdSelfUpdate.Flags()
+	flags.StringVar(&selfUpdateOptions.Output, "output", os.Args[0], "Save the downloaded file as `filename`")
+}
+
+func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
+	v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, Verbosef)
+	if err != nil {
+		return errors.Fatalf("unable to update restic: %v", err)
+	}
+
+	Printf("successfully updated restic to version %v\n", v)
+
+	return nil
+}

cmd/restic/cmd_snapshots.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
 
@@ -81,7 +82,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 		return nil
 	}
-	PrintSnapshots(gopts.stdout, list, opts.Compact)
+	PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
 
 	return nil
 }
@@ -123,7 +124,16 @@ func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
 }
 
 // PrintSnapshots prints a text table of the snapshots in list to stdout.
-func PrintSnapshots(stdout io.Writer, list restic.Snapshots, compact bool) {
+func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
+	// keep the reasons a snasphot is being kept in a map, so that it doesn't
+	// get lost when the list of snapshots is sorted
+	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
+	if len(reasons) > 0 {
+		for i, sn := range list {
+			id := sn.ID()
+			keepReasons[*id] = reasons[i]
+		}
+	}
 
 	// always sort the snapshots so that the newer ones are listed last
 	sort.SliceStable(list, func(i, j int) bool {
@@ -143,71 +153,72 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, compact bool) {
 		}
 	}
 
-	tab := NewTable()
-	if !compact {
-		tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
-		tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
+	tab := table.New()
+
+	if compact {
+		tab.AddColumn("ID", "{{ .ID }}")
+		tab.AddColumn("Time", "{{ .Timestamp }}")
+		tab.AddColumn("Host", "{{ .Hostname }}")
+		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
 	} else {
-		tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags")
-		tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%s", -maxHost)
+		tab.AddColumn("ID", "{{ .ID }}")
+		tab.AddColumn("Time", "{{ .Timestamp }}")
+		tab.AddColumn("Host      ", "{{ .Hostname }}")
+		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
+		if len(reasons) > 0 {
+			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
+		}
+		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
 	}
 
+	type snapshot struct {
+		ID        string
+		Timestamp string
+		Hostname  string
+		Tags      []string
+		Reasons   []string
+		Paths     []string
+	}
+
+	var multiline bool
 	for _, sn := range list {
-		if len(sn.Paths) == 0 {
-			continue
+		data := snapshot{
+			ID:        sn.ID().Str(),
+			Timestamp: sn.Time.Format(TimeFormat),
+			Hostname:  sn.Hostname,
+			Tags:      sn.Tags,
+			Paths:     sn.Paths,
 		}
 
-		firstTag := ""
-		if len(sn.Tags) > 0 {
-			firstTag = sn.Tags[0]
+		if len(reasons) > 0 {
+			id := sn.ID()
+			data.Reasons = keepReasons[*id].Matches
 		}
 
-		rows := len(sn.Paths)
-		if rows < len(sn.Tags) {
-			rows = len(sn.Tags)
+		if len(sn.Paths) > 1 {
+			multiline = true
 		}
 
-		treeElement := "   "
-		if rows != 1 {
-			treeElement = "┌──"
-		}
-
-		if !compact {
-			tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
-		} else {
-			allTags := ""
-			for _, tag := range sn.Tags {
-				allTags += tag + " "
-			}
-			tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, allTags})
-			continue
-		}
-
-		if len(sn.Tags) > rows {
-			rows = len(sn.Tags)
-		}
-
-		for i := 1; i < rows; i++ {
-			path := ""
-			if len(sn.Paths) > i {
-				path = sn.Paths[i]
-			}
-
-			tag := ""
-			if len(sn.Tags) > i {
-				tag = sn.Tags[i]
-			}
-
-			treeElement := "│"
-			if i == (rows - 1) {
-				treeElement = "└──"
-			}
-
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, treeElement, path})
-		}
+		tab.AddRow(data)
 	}
 
-	tab.Footer = fmt.Sprintf("%d snapshots", len(list))
+	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))
+
+	if multiline {
+		// print an additional blank line between snapshots
+
+		var last int
+		tab.PrintData = func(w io.Writer, idx int, s string) error {
+			var err error
+			if idx == last {
+				_, err = fmt.Fprintf(w, "%s\n", s)
+			} else {
+				_, err = fmt.Fprintf(w, "\n%s\n", s)
+			}
+			last = idx
+			return err
+		}
+	}
 
 	tab.Write(stdout)
 }

cmd/restic/cmd_stats.go

@@ -0,0 +1,324 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
+	"github.com/spf13/cobra"
+)
+
+var cmdStats = &cobra.Command{
+	Use:   "stats [flags] [snapshot-ID]",
+	Short: "Scan the repository and show basic statistics",
+	Long: `
+The "stats" command walks one or all snapshots in a repository and
+accumulates statistics about the data stored therein. It reports on
+the number of unique files and their sizes, according to one of
+the counting modes as given by the --mode flag.
+
+If no snapshot is specified, all snapshots will be considered. Some
+modes make more sense over just a single snapshot, while others
+are useful across all snapshots, depending on what you are trying
+to calculate.
+
+The modes are:
+
+* restore-size: (default) Counts the size of the restored files.
+* files-by-contents: Counts total size of files, where a file is
+   considered unique if it has unique contents.
+* raw-data: Counts the size of blobs in the repository, regardless of
+  how many files reference them.
+* blobs-per-file: A combination of files-by-contents and raw-data.
+* Refer to the online manual for more details about each mode.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runStats(globalOptions, args)
+	},
+}
+
+func init() {
+	cmdRoot.AddCommand(cmdStats)
+	f := cmdStats.Flags()
+	f.StringVar(&countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file, or raw-data")
+	f.StringVar(&snapshotByHost, "host", "H", "filter latest snapshot by this hostname")
+}
+
+func runStats(gopts GlobalOptions, args []string) error {
+	err := verifyStatsInput(gopts, args)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if err = repo.LoadIndex(ctx); err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		lock, err := lockRepo(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !gopts.JSON {
+		Printf("scanning...\n")
+	}
+
+	// create a container for the stats (and other needed state)
+	stats := &statsContainer{
+		uniqueFiles: make(map[fileID]struct{}),
+		fileBlobs:   make(map[string]restic.IDSet),
+		blobs:       restic.NewBlobSet(),
+		blobsSeen:   restic.NewBlobSet(),
+	}
+
+	if snapshotIDString != "" {
+		// scan just a single snapshot
+
+		var sID restic.ID
+		if snapshotIDString == "latest" {
+			sID, err = restic.FindLatestSnapshot(ctx, repo, []string{}, []restic.TagList{}, snapshotByHost)
+			if err != nil {
+				return errors.Fatalf("latest snapshot for criteria not found: %v", err)
+			}
+		} else {
+			sID, err = restic.FindSnapshot(repo, snapshotIDString)
+			if err != nil {
+				return errors.Fatalf("error loading snapshot: %v", err)
+			}
+		}
+
+		snapshot, err := restic.LoadSnapshot(ctx, repo, sID)
+		if err != nil {
+			return errors.Fatalf("error loading snapshot from repo: %v", err)
+		}
+
+		err = statsWalkSnapshot(ctx, snapshot, repo, stats)
+	} else {
+		// iterate every snapshot in the repo
+		err = repo.List(ctx, restic.SnapshotFile, func(snapshotID restic.ID, size int64) error {
+			snapshot, err := restic.LoadSnapshot(ctx, repo, snapshotID)
+			if err != nil {
+				return fmt.Errorf("Error loading snapshot %s: %v", snapshotID.Str(), err)
+			}
+			return statsWalkSnapshot(ctx, snapshot, repo, stats)
+		})
+	}
+	if err != nil {
+		return err
+	}
+
+	if countMode == countModeRawData {
+		// the blob handles have been collected, but not yet counted
+		for blobHandle := range stats.blobs {
+			blobSize, found := repo.LookupBlobSize(blobHandle.ID, blobHandle.Type)
+			if !found {
+				return fmt.Errorf("blob %v not found", blobHandle)
+			}
+			stats.TotalSize += uint64(blobSize)
+			stats.TotalBlobCount++
+		}
+	}
+
+	if gopts.JSON {
+		err = json.NewEncoder(os.Stdout).Encode(stats)
+		if err != nil {
+			return fmt.Errorf("encoding output: %v", err)
+		}
+		return nil
+	}
+
+	// inform the user what was scanned and how it was scanned
+	snapshotsScanned := snapshotIDString
+	if snapshotsScanned == "latest" {
+		snapshotsScanned = "the latest snapshot"
+	} else if snapshotsScanned == "" {
+		snapshotsScanned = "all snapshots"
+	}
+	Printf("Stats for %s in %s mode:\n", snapshotsScanned, countMode)
+
+	if stats.TotalBlobCount > 0 {
+		Printf("  Total Blob Count:   %d\n", stats.TotalBlobCount)
+	}
+	if stats.TotalFileCount > 0 {
+		Printf("  Total File Count:   %d\n", stats.TotalFileCount)
+	}
+	Printf("        Total Size:   %-5s\n", formatBytes(stats.TotalSize))
+
+	return nil
+}
+
+func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, stats *statsContainer) error {
+	if snapshot.Tree == nil {
+		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
+	}
+
+	if countMode == countModeRawData {
+		// count just the sizes of unique blobs; we don't need to walk the tree
+		// ourselves in this case, since a nifty function does it for us
+		return restic.FindUsedBlobs(ctx, repo, *snapshot.Tree, stats.blobs, stats.blobsSeen)
+	}
+
+	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats))
+	if err != nil {
+		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
+	}
+	return nil
+}
+
+func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFunc {
+	return func(_ restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
+		if nodeErr != nil {
+			return true, nodeErr
+		}
+		if node == nil {
+			return true, nil
+		}
+
+		if countMode == countModeUniqueFilesByContents || countMode == countModeBlobsPerFile {
+			// only count this file if we haven't visited it before
+			fid := makeFileIDByContents(node)
+			if _, ok := stats.uniqueFiles[fid]; !ok {
+				// mark the file as visited
+				stats.uniqueFiles[fid] = struct{}{}
+
+				if countMode == countModeUniqueFilesByContents {
+					// simply count the size of each unique file (unique by contents only)
+					stats.TotalSize += node.Size
+					stats.TotalFileCount++
+				}
+				if countMode == countModeBlobsPerFile {
+					// count the size of each unique blob reference, which is
+					// by unique file (unique by contents and file path)
+					for _, blobID := range node.Content {
+						// ensure we have this file (by path) in our map; in this
+						// mode, a file is unique by both contents and path
+						nodePath := filepath.Join(npath, node.Name)
+						if _, ok := stats.fileBlobs[nodePath]; !ok {
+							stats.fileBlobs[nodePath] = restic.NewIDSet()
+							stats.TotalFileCount++
+						}
+						if _, ok := stats.fileBlobs[nodePath][blobID]; !ok {
+							// is always a data blob since we're accessing it via a file's Content array
+							blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob)
+							if !found {
+								return true, fmt.Errorf("blob %s not found for tree %s", blobID, *node.Subtree)
+							}
+
+							// count the blob's size, then add this blob by this
+							// file (path) so we don't double-count it
+							stats.TotalSize += uint64(blobSize)
+							stats.fileBlobs[nodePath].Insert(blobID)
+							// this mode also counts total unique blob _references_ per file
+							stats.TotalBlobCount++
+						}
+					}
+				}
+			}
+		}
+
+		if countMode == countModeRestoreSize {
+			// as this is a file in the snapshot, we can simply count its
+			// size without worrying about uniqueness, since duplicate files
+			// will still be restored
+			stats.TotalSize += node.Size
+			stats.TotalFileCount++
+		}
+
+		return true, nil
+	}
+}
+
+// makeFileIDByContents returns a hash of the blob IDs of the
+// node's Content in sequence.
+func makeFileIDByContents(node *restic.Node) fileID {
+	var bb []byte
+	for _, c := range node.Content {
+		bb = append(bb, []byte(c[:])...)
+	}
+	return sha256.Sum256(bb)
+}
+
+func verifyStatsInput(gopts GlobalOptions, args []string) error {
+	// require a recognized counting mode
+	switch countMode {
+	case countModeRestoreSize:
+	case countModeUniqueFilesByContents:
+	case countModeBlobsPerFile:
+	case countModeRawData:
+	default:
+		return fmt.Errorf("unknown counting mode: %s (use the -h flag to get a list of supported modes)", countMode)
+	}
+
+	// ensure at most one snapshot was specified
+	if len(args) > 1 {
+		return fmt.Errorf("only one snapshot may be specified")
+	}
+
+	// if a snapshot was specified, mark it as the one to scan
+	if len(args) == 1 {
+		snapshotIDString = args[0]
+	}
+
+	return nil
+}
+
+// statsContainer holds information during a walk of a repository
+// to collect information about it, as well as state needed
+// for a successful and efficient walk.
+type statsContainer struct {
+	TotalSize      uint64 `json:"total_size"`
+	TotalFileCount uint64 `json:"total_file_count"`
+	TotalBlobCount uint64 `json:"total_blob_count,omitempty"`
+
+	// uniqueFiles marks visited files according to their
+	// contents (hashed sequence of content blob IDs)
+	uniqueFiles map[fileID]struct{}
+
+	// fileBlobs maps a file name (path) to the set of
+	// blobs that have been seen as a part of the file
+	fileBlobs map[string]restic.IDSet
+
+	// blobs and blobsSeen are used to count indiviudal
+	// unique blobs, independent of references to files
+	blobs, blobsSeen restic.BlobSet
+}
+
+// fileID is a 256-bit hash that distinguishes unique files.
+type fileID [32]byte
+
+var (
+	// the mode of counting to perform
+	countMode string
+
+	// the snapshot to scan, as given by the user
+	snapshotIDString string
+
+	// snapshotByHost is the host to filter latest
+	// snapshot by, if given by user
+	snapshotByHost string
+)
+
+const (
+	countModeRestoreSize           = "restore-size"
+	countModeUniqueFilesByContents = "files-by-contents"
+	countModeBlobsPerFile          = "blobs-per-file"
+	countModeRawData               = "raw-data"
+)

cmd/restic/cmd_version.go

@@ -16,7 +16,7 @@ and the version of this software.
 `,
 	DisableAutoGenTag: true,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Printf("restic %s\ncompiled with %v on %v/%v\n",
+		fmt.Printf("restic %s compiled with %v on %v/%v\n",
 			version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
 	},
 }

cmd/restic/exclude.go

@@ -60,15 +60,20 @@ func (rc *rejectionCache) Store(dir string, rejected bool) {
 	rc.m[dir] = rejected
 }
 
+// RejectByNameFunc is a function that takes a filename of a
+// file that would be included in the backup. The function returns true if it
+// should be excluded (rejected) from the backup.
+type RejectByNameFunc func(path string) bool
+
 // RejectFunc is a function that takes a filename and os.FileInfo of a
 // file that would be included in the backup. The function returns true if it
 // should be excluded (rejected) from the backup.
 type RejectFunc func(path string, fi os.FileInfo) bool
 
-// rejectByPattern returns a RejectFunc which rejects files that match
+// rejectByPattern returns a RejectByNameFunc which rejects files that match
 // one of the patterns.
-func rejectByPattern(patterns []string) RejectFunc {
-	return func(item string, fi os.FileInfo) bool {
+func rejectByPattern(patterns []string) RejectByNameFunc {
+	return func(item string) bool {
 		matched, _, err := filter.List(patterns, item)
 		if err != nil {
 			Warnf("error for exclude pattern: %v", err)
@@ -83,14 +88,14 @@ func rejectByPattern(patterns []string) RejectFunc {
 	}
 }
 
-// rejectIfPresent returns a RejectFunc which itself returns whether a path
-// should be excluded. The RejectFunc considers a file to be excluded when
+// rejectIfPresent returns a RejectByNameFunc which itself returns whether a path
+// should be excluded. The RejectByNameFunc considers a file to be excluded when
 // it resides in a directory with an exclusion file, that is specified by
 // excludeFileSpec in the form "filename[:content]". The returned error is
 // non-nil if the filename component of excludeFileSpec is empty. If rc is
-// non-nil, it is going to be used in the RejectFunc to expedite the evaluation
+// non-nil, it is going to be used in the RejectByNameFunc to expedite the evaluation
 // of a directory based on previous visits.
-func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) {
+func rejectIfPresent(excludeFileSpec string) (RejectByNameFunc, error) {
 	if excludeFileSpec == "" {
 		return nil, errors.New("name for exclusion tagfile is empty")
 	}
@@ -107,7 +112,7 @@ func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) {
 	}
 	debug.Log("using %q as exclusion tagfile", tf)
 	rc := &rejectionCache{}
-	fn := func(filename string, _ os.FileInfo) bool {
+	fn := func(filename string) bool {
 		return isExcludedByFile(filename, tf, tc, rc)
 	}
 	return fn, nil
@@ -185,6 +190,11 @@ func isDirExcludedByFile(dir, tagFilename, header string) bool {
 func gatherDevices(items []string) (deviceMap map[string]uint64, err error) {
 	deviceMap = make(map[string]uint64)
 	for _, item := range items {
+		item, err = filepath.Abs(filepath.Clean(item))
+		if err != nil {
+			return nil, err
+		}
+
 		fi, err := fs.Lstat(item)
 		if err != nil {
 			return nil, err
@@ -215,6 +225,8 @@ func rejectByDevice(samples []string) (RejectFunc, error) {
 			return false
 		}
 
+		item = filepath.Clean(item)
+
 		id, err := fs.DeviceID(fi)
 		if err != nil {
 			// This should never happen because gatherDevices() would have
@@ -222,11 +234,14 @@ func rejectByDevice(samples []string) (RejectFunc, error) {
 			panic(err)
 		}
 
-		for dir := item; dir != ""; dir = filepath.Dir(dir) {
+		for dir := item; ; dir = filepath.Dir(dir) {
 			debug.Log("item %v, test dir %v", item, dir)
 
 			allowedID, ok := allowed[dir]
 			if !ok {
+				if dir == filepath.Dir(dir) {
+					break
+				}
 				continue
 			}
 
@@ -242,11 +257,11 @@ func rejectByDevice(samples []string) (RejectFunc, error) {
 	}, nil
 }
 
-// rejectResticCache returns a RejectFunc that rejects the restic cache
+// rejectResticCache returns a RejectByNameFunc that rejects the restic cache
 // directory (if set).
-func rejectResticCache(repo *repository.Repository) (RejectFunc, error) {
+func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
 	if repo.Cache == nil {
-		return func(string, os.FileInfo) bool {
+		return func(string) bool {
 			return false
 		}, nil
 	}
@@ -256,7 +271,7 @@ func rejectResticCache(repo *repository.Repository) (RejectFunc, error) {
 		return nil, errors.New("cacheBase is empty string")
 	}
 
-	return func(item string, _ os.FileInfo) bool {
+	return func(item string) bool {
 		if fs.HasPathPrefix(cacheBase, item) {
 			debug.Log("rejecting restic cache directory %v", item)
 			return true

cmd/restic/exclude_test.go

@@ -27,7 +27,7 @@ func TestRejectByPattern(t *testing.T) {
 	for _, tc := range tests {
 		t.Run("", func(t *testing.T) {
 			reject := rejectByPattern(patterns)
-			res := reject(tc.filename, nil)
+			res := reject(tc.filename)
 			if res != tc.reject {
 				t.Fatalf("wrong result for filename %v: want %v, got %v",
 					tc.filename, tc.reject, res)
@@ -140,8 +140,8 @@ func TestMultipleIsExcludedByFile(t *testing.T) {
 		if err != nil {
 			return err
 		}
-		excludedByFoo := fooExclude(p, fi)
-		excludedByBar := barExclude(p, fi)
+		excludedByFoo := fooExclude(p)
+		excludedByBar := barExclude(p)
 		excluded := excludedByFoo || excludedByBar
 		// the log message helps debugging in case the test fails
 		t.Logf("%q: %v || %v = %v", p, excludedByFoo, excludedByBar, excluded)

cmd/restic/excludes

@@ -1,31 +0,0 @@
-/boot
-/dev
-/etc
-/home
-/lost+found
-/mnt
-/proc
-/root
-/run
-/sys
-/tmp
-/usr
-/var
-/opt/android-sdk
-/opt/bullet
-/opt/dex2jar
-/opt/jameica
-/opt/google
-/opt/JDownloader
-/opt/JDownloaderScripts
-/opt/opencascade
-/opt/vagrant
-/opt/visual-studio-code
-/opt/vtk6
-/bin
-/fonts*
-/srv/ftp
-/srv/http
-/sbin
-/lib
-/lib64

cmd/restic/format.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/restic/restic/internal/restic"
@@ -63,9 +62,9 @@ func formatDuration(d time.Duration) string {
 	return formatSeconds(sec)
 }
 
-func formatNode(prefix string, n *restic.Node, long bool) string {
+func formatNode(path string, n *restic.Node, long bool) string {
 	if !long {
-		return filepath.Join(prefix, n.Name)
+		return path
 	}
 
 	var mode os.FileMode
@@ -91,6 +90,6 @@ func formatNode(prefix string, n *restic.Node, long bool) string {
 
 	return fmt.Sprintf("%s %5d %5d %6d %s %s%s",
 		mode|n.Mode, n.UID, n.GID, n.Size,
-		n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name),
+		n.ModTime.Format(TimeFormat), path,
 		target)
 }