Add patched chunker

Referenced in https://forum.restic.net/t/restic-slice-bounds-out-of-range/2617/20

.github/ISSUE_TEMPLATE/Bug.md

@@ -83,8 +83,8 @@ Do you have an idea how to solve the issue?
 
 
 
-Did restic help you or made you happy in any way?
--------------------------------------------------
+Did restic help you today? Did it make you happy in any way?
+------------------------------------------------------------
 
 <!--
 Answering this question is not required, but if you have anything positive to share, please do so here!

.github/ISSUE_TEMPLATE/Feature.md

@@ -47,8 +47,8 @@ This section should contain a brief description what you're trying to do, which
 would be possible after implementing the new feature.
 -->
 
-Did restic help you or made you happy in any way?
--------------------------------------------------
+Did restic help you today? Did it make you happy in any way?
+------------------------------------------------------------
 
 <!--
 Answering this question is not required, but if you have anything positive to share, please do so here!

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,4 @@
+contact_links:
+  - name: restic forum
+    url: https://forum.restic.net
+    about: Please ask questions about using restic here, do not open an issue for questions.

.github/PULL_REQUEST_TEMPLATE.md

@@ -22,12 +22,16 @@ Was the change discussed in an issue or in the forum before?
 
 <!--
 Link issues and relevant forum posts here.
+
+If this PR resolves an issue on GitHub, use "closes #1234" so that the issue is
+closed automatically when this PR is merged.
 -->
 
 Checklist
 ---------
 
 - [ ] I have read the [Contribution Guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches)
+- [ ] I have enabled [maintainer edits for this PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork)
 - [ ] I have added tests for all changes in this PR
 - [ ] I have added documentation for the changes (in the manual)
 - [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (template [here](https://github.com/restic/restic/blob/master/changelog/TEMPLATE))

.gitignore

@@ -1,3 +1,2 @@
 /restic
 /.vagrant
-/doc/_build

.travis.yml

@@ -4,17 +4,45 @@ sudo: false
 matrix:
   include:
     - os: linux
-      go: "1.9.x"
-      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0 RESTIC_BUILD_SOLARIS=0
+      go: "1.11.x"
+      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
+
+    - os: linux
+      go: "1.12.x"
+      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
+
+    - os: linux
+      go: "1.13.x"
+      env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
 
     # only run fuse and cloud backends tests on Travis for the latest Go on Linux
     - os: linux
-      go: "1.10.x"
+      go: "1.14.x"
       sudo: true
+      cache:
+        directories:
+          - $HOME/.cache/go-build
+          - $HOME/gopath/pkg/mod
 
     - os: osx
-      go: "1.10.x"
+      go: "1.14.x"
       env: RESTIC_TEST_FUSE=0 RESTIC_TEST_CLOUD_BACKENDS=0
+      cache:
+        directories:
+          - $HOME/Library/Caches/go-build
+          - $HOME/gopath/pkg/mod
 
 branches:
   only:
@@ -36,6 +64,3 @@ install:
 
 script:
   - go run run_integration_tests.go
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash) -f all.cov

CHANGELOG.md

@@ -1,3 +1,547 @@
+Changelog for restic 0.9.6 (2019-11-22)
+=======================================
+
+The following sections list the changes in restic 0.9.6 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2063: Allow absolute path for filename when backing up from stdin
+ * Fix #2174: Save files with invalid timestamps
+ * Fix #2249: Read fresh metadata for unmodified files
+ * Fix #2301: Add upper bound for t in --read-data-subset=n/t
+ * Fix #2321: Check errors when loading index files
+ * Enh #2179: Use ctime when checking for file changes
+ * Enh #2306: Allow multiple retries for interactive password input
+ * Enh #2330: Make `--group-by` accept both singular and plural
+ * Enh #2350: Add option to configure S3 region
+
+Details
+-------
+
+ * Bugfix #2063: Allow absolute path for filename when backing up from stdin
+
+   When backing up from stdin, handle directory path for `--stdin-filename`. This can be used to
+   specify the full path for the backed-up file.
+
+   https://github.com/restic/restic/issues/2063
+
+ * Bugfix #2174: Save files with invalid timestamps
+
+   When restic reads invalid timestamps (year is before 0000 or after 9999) it refused to read and
+   archive the file. We've changed the behavior and will now save modified timestamps with the
+   year set to either 0000 or 9999, the rest of the timestamp stays the same, so the file will be saved
+   (albeit with a bogus timestamp).
+
+   https://github.com/restic/restic/issues/2174
+   https://github.com/restic/restic/issues/1173
+
+ * Bugfix #2249: Read fresh metadata for unmodified files
+
+   Restic took all metadata for files which were detected as unmodified, not taking into account
+   changed metadata (ownership, mode). This is now corrected.
+
+   https://github.com/restic/restic/issues/2249
+   https://github.com/restic/restic/pull/2252
+
+ * Bugfix #2301: Add upper bound for t in --read-data-subset=n/t
+
+   256 is the effective maximum for t, but restic would allow larger values, leading to strange
+   behavior.
+
+   https://github.com/restic/restic/issues/2301
+   https://github.com/restic/restic/pull/2304
+
+ * Bugfix #2321: Check errors when loading index files
+
+   Restic now checks and handles errors which occur when loading index files, the missing check
+   leads to odd errors (and a stack trace printed to users) later. This was reported in the forum.
+
+   https://github.com/restic/restic/pull/2321
+   https://forum.restic.net/t/check-rebuild-index-prune/1848/13
+
+ * Enhancement #2179: Use ctime when checking for file changes
+
+   Previously, restic only checked a file's mtime (along with other non-timestamp metadata) to
+   decide if a file has changed. This could cause restic to not notice that a file has changed (and
+   therefore continue to store the old version, as opposed to the modified version) if something
+   edits the file and then resets the timestamp. Restic now also checks the ctime of files, so any
+   modifications to a file should be noticed, and the modified file will be backed up. The ctime
+   check will be disabled if the --ignore-inode flag was given.
+
+   If this change causes problems for you, please open an issue, and we can look in to adding a
+   seperate flag to disable just the ctime check.
+
+   https://github.com/restic/restic/issues/2179
+   https://github.com/restic/restic/pull/2212
+
+ * Enhancement #2306: Allow multiple retries for interactive password input
+
+   Restic used to quit if the repository password was typed incorrectly once. Restic will now ask
+   the user again for the repository password if typed incorrectly. The user will now get three
+   tries to input the correct password before restic quits.
+
+   https://github.com/restic/restic/issues/2306
+
+ * Enhancement #2330: Make `--group-by` accept both singular and plural
+
+   One can now use the values `host`/`hosts`, `path`/`paths` and `tag` / `tags` interchangeably
+   in the `--group-by` argument.
+
+   https://github.com/restic/restic/issues/2330
+
+ * Enhancement #2350: Add option to configure S3 region
+
+   We've added a new option for setting the region when accessing an S3-compatible service. For
+   some providers, it is required to set this to a valid value. You can do that either by setting the
+   environment variable `AWS_DEFAULT_REGION` or using the option `s3.region`, e.g. like this:
+   `-o s3.region="us-east-1"`.
+
+   https://github.com/restic/restic/pull/2350
+
+
+Changelog for restic 0.9.5 (2019-04-23)
+=======================================
+
+The following sections list the changes in restic 0.9.5 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2135: Return error when no bytes could be read from stdin
+ * Fix #2181: Don't cancel timeout after 30 seconds for self-update
+ * Fix #2203: Fix reading passwords from stdin
+ * Fix #2224: Don't abort the find command when a tree can't be loaded
+ * Enh #1895: Add case insensitive include & exclude options
+ * Enh #1937: Support streaming JSON output for backup
+ * Enh #2155: Add Openstack application credential auth for Swift
+ * Enh #2184: Add --json support to forget command
+ * Enh #2037: Add group-by option to snapshots command
+ * Enh #2124: Ability to dump folders to tar via stdout
+ * Enh #2139: Return error if no bytes could be read for `backup --stdin`
+ * Enh #2205: Add --ignore-inode option to backup cmd
+ * Enh #2220: Add config option to set S3 storage class
+
+Details
+-------
+
+ * Bugfix #2135: Return error when no bytes could be read from stdin
+
+   We assume that users reading backup data from stdin want to know when no data could be read, so now
+   restic returns an error when `backup --stdin` is called but no bytes could be read. Usually,
+   this means that an earlier command in a pipe has failed. The documentation was amended and now
+   recommends setting the `pipefail` option (`set -o pipefail`).
+
+   https://github.com/restic/restic/pull/2135
+   https://github.com/restic/restic/pull/2139
+
+ * Bugfix #2181: Don't cancel timeout after 30 seconds for self-update
+
+   https://github.com/restic/restic/issues/2181
+
+ * Bugfix #2203: Fix reading passwords from stdin
+
+   Passwords for the `init`, `key add`, and `key passwd` commands can now be read from
+   non-terminal stdin.
+
+   https://github.com/restic/restic/issues/2203
+
+ * Bugfix #2224: Don't abort the find command when a tree can't be loaded
+
+   Change the find command so that missing trees don't result in a crash. Instead, the error is
+   logged to the debug log, and the tree ID is displayed along with the snapshot it belongs to. This
+   makes it possible to recover repositories that are missing trees by forgetting the snapshots
+   they are used in.
+
+   https://github.com/restic/restic/issues/2224
+
+ * Enhancement #1895: Add case insensitive include & exclude options
+
+   The backup and restore commands now have --iexclude and --iinclude flags as case insensitive
+   variants of --exclude and --include.
+
+   https://github.com/restic/restic/issues/1895
+   https://github.com/restic/restic/pull/2032
+
+ * Enhancement #1937: Support streaming JSON output for backup
+
+   We've added support for getting machine-readable status output during backup, just pass the
+   flag `--json` for `restic backup` and restic will output a stream of JSON objects which contain
+   the current progress.
+
+   https://github.com/restic/restic/issues/1937
+   https://github.com/restic/restic/pull/1944
+
+ * Enhancement #2155: Add Openstack application credential auth for Swift
+
+   Since Openstack Queens Identity (auth V3) service supports an application credential auth
+   method. It allows to create a technical account with the limited roles. This commit adds an
+   application credential authentication method for the Swift backend.
+
+   https://github.com/restic/restic/issues/2155
+
+ * Enhancement #2184: Add --json support to forget command
+
+   The forget command now supports the --json argument, outputting the information about what is
+   (or would-be) kept and removed from the repository.
+
+   https://github.com/restic/restic/issues/2184
+   https://github.com/restic/restic/pull/2185
+
+ * Enhancement #2037: Add group-by option to snapshots command
+
+   We have added an option to group the output of the snapshots command, similar to the output of the
+   forget command. The option has been called "--group-by" and accepts any combination of the
+   values "host", "paths" and "tags", separated by commas. Default behavior (not specifying
+   --group-by) has not been changed. We have added support of the grouping to the JSON output.
+
+   https://github.com/restic/restic/issues/2037
+   https://github.com/restic/restic/pull/2087
+
+ * Enhancement #2124: Ability to dump folders to tar via stdout
+
+   We've added the ability to dump whole folders to stdout via the `dump` command. Restic now
+   requires at least Go 1.10 due to a limitation of the standard library for Go <= 1.9.
+
+   https://github.com/restic/restic/issues/2123
+   https://github.com/restic/restic/pull/2124
+
+ * Enhancement #2139: Return error if no bytes could be read for `backup --stdin`
+
+   When restic is used to backup the output of a program, like `mysqldump | restic backup --stdin`,
+   it now returns an error if no bytes could be read at all. This catches the failure case when
+   `mysqldump` failed for some reason and did not output any data to stdout.
+
+   https://github.com/restic/restic/pull/2139
+
+ * Enhancement #2205: Add --ignore-inode option to backup cmd
+
+   This option handles backup of virtual filesystems that do not keep fixed inodes for files, like
+   Fuse-based, pCloud, etc. Ignoring inode changes allows to consider the file as unchanged if
+   last modification date and size are unchanged.
+
+   https://github.com/restic/restic/issues/1631
+   https://github.com/restic/restic/pull/2205
+   https://github.com/restic/restic/pull/2047
+
+ * Enhancement #2220: Add config option to set S3 storage class
+
+   The `s3.storage-class` option can be passed to restic (using `-o`) to specify the storage
+   class to be used for S3 objects created by restic.
+
+   The storage class is passed as-is to S3, so it needs to be understood by the API. On AWS, it can be
+   one of `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING` and
+   `REDUCED_REDUNDANCY`. If unspecified, the default storage class is used (`STANDARD` on
+   AWS).
+
+   You can mix storage classes in the same bucket, and the setting isn't stored in the restic
+   repository, so be sure to specify it with each command that writes to S3.
+
+   https://github.com/restic/restic/issues/706
+   https://github.com/restic/restic/pull/2220
+
+
+Changelog for restic 0.9.4 (2019-01-06)
+=======================================
+
+The following sections list the changes in restic 0.9.4 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1989: Google Cloud Storage: Respect bandwidth limit
+ * Fix #2040: Add host name filter shorthand flag for `stats` command
+ * Fix #2068: Correctly return error loading data
+ * Fix #2095: Consistently use local time for snapshots times
+ * Enh #1605: Concurrent restore
+ * Enh #2089: Increase granularity of the "keep within" retention policy
+ * Enh #2097: Add key hinting
+ * Enh #2017: Mount: Enforce FUSE Unix permissions with allow-other
+ * Enh #2070: Make all commands display timestamps in local time
+ * Enh #2085: Allow --files-from to be specified multiple times
+ * Enh #2094: Run command to get password
+
+Details
+-------
+
+ * Bugfix #1989: Google Cloud Storage: Respect bandwidth limit
+
+   The GCS backend did not respect the bandwidth limit configured, a previous commit
+   accidentally removed support for it.
+
+   https://github.com/restic/restic/issues/1989
+   https://github.com/restic/restic/pull/2100
+
+ * Bugfix #2040: Add host name filter shorthand flag for `stats` command
+
+   The default value for `--host` flag was set to 'H' (the shorthand version of the flag), this
+   caused the lookup for the latest snapshot to fail.
+
+   Add shorthand flag `-H` for `--host` (with empty default so if these flags are not specified the
+   latest snapshot will not filter by host name).
+
+   Also add shorthand `-H` for `backup` command.
+
+   https://github.com/restic/restic/issues/2040
+
+ * Bugfix #2068: Correctly return error loading data
+
+   In one case during `prune` and `check`, an error loading data from the backend is not returned
+   properly. This is now corrected.
+
+   https://github.com/restic/restic/issues/1999#issuecomment-433737921
+   https://github.com/restic/restic/pull/2068
+
+ * Bugfix #2095: Consistently use local time for snapshots times
+
+   By default snapshots created with restic backup were set to local time, but when the --time flag
+   was used the provided timestamp was parsed as UTC. With this change all snapshots times are set
+   to local time.
+
+   https://github.com/restic/restic/pull/2095
+
+ * Enhancement #1605: Concurrent restore
+
+   This change significantly improves restore performance, especially when using
+   high-latency remote repositories like B2.
+
+   The implementation now uses several concurrent threads to download and process multiple
+   remote files concurrently. To further reduce restore time, each remote file is downloaded
+   using a single repository request.
+
+   https://github.com/restic/restic/issues/1605
+   https://github.com/restic/restic/pull/1719
+
+ * Enhancement #2089: Increase granularity of the "keep within" retention policy
+
+   The `keep-within` option of the `forget` command now accepts time ranges with an hourly
+   granularity. For example, running `restic forget --keep-within 3d12h` will keep all the
+   snapshots made within three days and twelve hours from the time of the latest snapshot.
+
+   https://github.com/restic/restic/issues/2089
+   https://github.com/restic/restic/pull/2090
+
+ * Enhancement #2097: Add key hinting
+
+   Added a new option `--key-hint` and corresponding environment variable `RESTIC_KEY_HINT`.
+   The key hint is a key ID to try decrypting first, before other keys in the repository.
+
+   This change will benefit repositories with many keys; if the correct key hint is supplied then
+   restic only needs to check one key. If the key hint is incorrect (the key does not exist, or the
+   password is incorrect) then restic will check all keys, as usual.
+
+   https://github.com/restic/restic/issues/2097
+
+ * Enhancement #2017: Mount: Enforce FUSE Unix permissions with allow-other
+
+   The fuse mount (`restic mount`) now lets the kernel check the permissions of the files within
+   snapshots (this is done through the `DefaultPermissions` FUSE option) when the option
+   `--allow-other` is specified.
+
+   To restore the old behavior, we've added the `--no-default-permissions` option. This allows
+   all users that have access to the mount point to access all files within the snapshots.
+
+   https://github.com/restic/restic/pull/2017
+
+ * Enhancement #2070: Make all commands display timestamps in local time
+
+   Restic used to drop the timezone information from displayed timestamps, it now converts
+   timestamps to local time before printing them so the times can be easily compared to.
+
+   https://github.com/restic/restic/pull/2070
+
+ * Enhancement #2085: Allow --files-from to be specified multiple times
+
+   Before, restic took only the last file specified with `--files-from` into account, this is now
+   corrected.
+
+   https://github.com/restic/restic/issues/2085
+   https://github.com/restic/restic/pull/2086
+
+ * Enhancement #2094: Run command to get password
+
+   We've added the `--password-command` option which allows specifying a command that restic
+   runs every time the password for the repository is needed, so it can be integrated with a
+   password manager or keyring. The option can also be set via the environment variable
+   `$RESTIC_PASSWORD_COMMAND`.
+
+   https://github.com/restic/restic/pull/2094
+
+
+Changelog for restic 0.9.3 (2018-10-13)
+=======================================
+
+The following sections list the changes in restic 0.9.3 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1935: Remove truncated files from cache
+ * Fix #1978: Do not return an error when the scanner is slower than backup
+ * Enh #1766: Restore: suppress lchown errors when not running as root
+ * Enh #1909: Reject files/dirs by name first
+ * Enh #1940: Add directory filter to ls command
+ * Enh #1967: Use `--host` everywhere
+ * Enh #2028: Display size of cache directories
+ * Enh #1777: Improve the `find` command
+ * Enh #1876: Display reason why forget keeps snapshots
+ * Enh #1891: Accept glob in paths loaded via --files-from
+ * Enh #1920: Vendor dependencies with Go 1.11 Modules
+ * Enh #1949: Add new command `self-update`
+ * Enh #1953: Ls: Add JSON output support for restic ls cmd
+ * Enh #1962: Stream JSON output for ls command
+
+Details
+-------
+
+ * Bugfix #1935: Remove truncated files from cache
+
+   When a file in the local cache is truncated, and restic tries to access data beyond the end of the
+   (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are
+   removed and the data is fetched directly from the backend.
+
+   https://github.com/restic/restic/issues/1935
+
+ * Bugfix #1978: Do not return an error when the scanner is slower than backup
+
+   When restic makes a backup, there's a background task called "scanner" which collects
+   information on how many files and directories are to be saved, in order to display progress
+   information to the user. When the backup finishes faster than the scanner, it is aborted
+   because the result is not needed any more. This logic contained a bug, where quitting the
+   scanner process was treated as an error, and caused restic to print an unhelpful error message
+   ("context canceled").
+
+   https://github.com/restic/restic/issues/1978
+   https://github.com/restic/restic/pull/1991
+
+ * Enhancement #1766: Restore: suppress lchown errors when not running as root
+
+   Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files
+   during restore if it is run as root, on non-Windows operating systems. On Windows, the error
+   is reported as usual.
+
+   https://github.com/restic/restic/issues/1766
+
+ * Enhancement #1909: Reject files/dirs by name first
+
+   The current scanner/archiver code had an architectural limitation: it always ran the
+   `lstat()` system call on all files and directories before a decision to include/exclude the
+   file/dir was made. This lead to a lot of unnecessary system calls for items that could have been
+   rejected by their name or path only.
+
+   We've changed the archiver/scanner implementation so that it now first rejects by name/path,
+   and only runs the system call on the remaining items. This reduces the number of `lstat()`
+   system calls a lot (depending on the exclude settings).
+
+   https://github.com/restic/restic/issues/1909
+   https://github.com/restic/restic/pull/1912
+
+ * Enhancement #1940: Add directory filter to ls command
+
+   The ls command can now be filtered by directories, so that only files in the given directories
+   will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list
+   their files as well.
+
+   It used to be possible to specify multiple snapshots, but that has been replaced by only one
+   snapshot and the possibility of specifying multiple directories.
+
+   Specifying directories constrains the walk, which can significantly speed up the listing.
+
+   https://github.com/restic/restic/issues/1940
+   https://github.com/restic/restic/pull/1941
+
+ * Enhancement #1967: Use `--host` everywhere
+
+   We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g.
+   for `restic backup`) still works, but will print a deprecation warning. Also, add the short
+   option `-H` where possible.
+
+   https://github.com/restic/restic/issues/1967
+
+ * Enhancement #2028: Display size of cache directories
+
+   The `cache` command now by default shows the size of the individual cache directories. It can be
+   disabled with `--no-size`.
+
+   https://github.com/restic/restic/issues/2028
+   https://github.com/restic/restic/pull/2033
+
+ * Enhancement #1777: Improve the `find` command
+
+   We've updated the `find` command to support multiple patterns.
+
+   `restic find` is now able to list the snapshots containing a specific tree or blob, or even the
+   snapshots that contain blobs belonging to a given pack. A list of IDs can be given, as long as they
+   all have the same type.
+
+   The command `find` can also display the pack IDs the blobs belong to, if the `--show-pack-id`
+   flag is provided.
+
+   https://github.com/restic/restic/issues/1777
+   https://github.com/restic/restic/pull/1780
+
+ * Enhancement #1876: Display reason why forget keeps snapshots
+
+   We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a
+   particuliar snapshot. This makes debugging policies for forget much easier. Please remember
+   to always try things out with `--dry-run`!
+
+   https://github.com/restic/restic/pull/1876
+
+ * Enhancement #1891: Accept glob in paths loaded via --files-from
+
+   Before that, behaviour was different if paths were appended to command line or from a file,
+   because wild card characters were expanded by shell if appended to command line, but not
+   expanded if loaded from file.
+
+   https://github.com/restic/restic/issues/1891
+
+ * Enhancement #1920: Vendor dependencies with Go 1.11 Modules
+
+   Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules.
+   For users this does not change much, only if you want to compile restic without downloading
+   anything with Go 1.11, then you need to run: `go build -mod=vendor build.go`
+
+   https://github.com/restic/restic/pull/1920
+
+ * Enhancement #1949: Add new command `self-update`
+
+   We have added a new command called `self-update` which downloads the latest released version
+   of restic from GitHub and replaces the current binary with it. It does not rely on any external
+   program (so it'll work everywhere), but still verifies the GPG signature using the embedded
+   GPG public key.
+
+   By default, the `self-update` command is hidden behind the `selfupdate` built tag, which is
+   only set when restic is built using `build.go` (including official releases). The reason for
+   this is that downstream distributions will then not include the command by default, so users
+   are encouraged to use the platform-specific distribution mechanism.
+
+   https://github.com/restic/restic/pull/1949
+
+ * Enhancement #1953: Ls: Add JSON output support for restic ls cmd
+
+   We've implemented listing files in the repository with JSON as output, just pass `--json` as an
+   option to `restic ls`. This makes the output of the command machine readable.
+
+   https://github.com/restic/restic/pull/1953
+
+ * Enhancement #1962: Stream JSON output for ls command
+
+   The `ls` command now supports JSON output with the global `--json` flag, and this change
+   streams out JSON messages one object at a time rather than en entire array buffered in memory
+   before encoding. The advantage is it allows large listings to be handled efficiently.
+
+   Two message types are printed: snapshots and nodes. A snapshot object will precede node
+   objects which belong to that snapshot. The `struct_type` field can be used to determine which
+   kind of message an object is.
+
+   https://github.com/restic/restic/pull/1962
+
+
 Changelog for restic 0.9.2 (2018-08-06)
 =======================================
 

CONTRIBUTING.md

@@ -46,8 +46,8 @@ Remember, the easier it is for us to reproduce the bug, the earlier it will be
 corrected!
 
 In addition, you can compile restic with debug support by running
-`go run build.go -tags debug` and instructing it to create a debug log by
-setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
+`go run build.go -tags debug` and instructing it to create a debug
+log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
 
     $ export DEBUG_LOG=/tmp/restic-debug.log
     $ restic backup ~/work
@@ -60,9 +60,37 @@ uploading it somewhere or post only the parts that are really relevant.
 Development Environment
 =======================
 
-In order to compile restic with the `go` tool directly, it needs to be checked
-out at the right path within a `GOPATH`. The concept of a `GOPATH` is explained
-in ["How to write Go code"](https://golang.org/doc/code.html).
+The repository contains several sets of directories with code: `cmd/` and
+`internal/` contain the code written for restic, whereas `vendor/` contains
+copies of libraries restic depends on. The libraries are managed with the
+command `go mod vendor`.
+
+Go >= 1.11
+----------
+
+For Go version 1.11 or later, you should clone the repo (without having
+`$GOPATH` set) and `cd` into the directory:
+
+    $ unset GOPATH
+    $ git clone https://github.com/restic/restic
+    $ cd restic
+
+Then use the `go` tool to build restic:
+
+    $ go build ./cmd/restic
+    $ ./restic version
+    restic 0.9.2-dev (compiled manually) compiled with go1.11 on linux/amd64
+
+You can run all tests with the following command:
+
+    $ go test ./...
+
+Go < 1.11
+---------
+
+In order to compile restic with Go before 1.11, it needs to be checked out at
+the right path within a `GOPATH`. The concept of a `GOPATH` is explained in
+["How to write Go code"](https://golang.org/doc/code.html).
 
 If you do not have a directory with Go code yet, executing the following
 instructions in your shell will create one for you and check out the restic
@@ -83,12 +111,7 @@ You can then build restic as follows:
 
 The following commands can be used to run all the tests:
 
-    $ go test ./cmd/... ./internal/...
-
-The repository contains two sets of directories with code: `cmd/` and
-`internal/` contain the code written for restic, whereas `vendor/` contains
-copies of libraries restic depends on. The libraries are managed with the
-[`dep`](https://github.com/golang/dep) tool.
+    $ go test ./...
 
 Providing Patches
 =================
@@ -107,8 +130,7 @@ down to the following steps:
 
  2. Clone the repository locally and create a new branch. If you are working on
     the code itself, please set up the development environment as described in
-    the previous section. Especially take care to place your forked repository
-    at the correct path (`src/github.com/restic/restic`) within your `GOPATH`.
+    the previous section.
 
  3. Then commit your changes as fine grained as possible, as smaller patches,
     that handle one and only one issue are easier to discuss and merge.
@@ -116,7 +138,7 @@ down to the following steps:
  4. Push the new branch with your changes to your fork of the repository.
 
  5. Create a pull request by visiting the GitHub website, it will guide you
-    through the process.
+    through the process. Please [allow edits from maintainers](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
 
  6. You will receive comments on your code and the feature or bug that they
     address. Maybe you need to rework some minor things, in this case push new
@@ -124,11 +146,14 @@ down to the following steps:
     existing commit, use common sense to decide which is better), they will be
     automatically added to the pull request.
 
- 7. If your pull request changes anything that users should be aware of (a
-    bugfix, a new feature, ...) please add an entry to the file
-    ['CHANGELOG.md'](CHANGELOG.md). It will be used in the announcement of the
-    next stable release. While writing, ask yourself: If I were the user, what
-    would I need to be aware of with this change.
+ 7. If your pull request changes anything that users should be aware
+    of (a bugfix, a new feature, ...) please add an entry as a new
+    file in `changelog/unreleased` including the issue number in the
+    filename (e.g. `issue-8756`). Use the template in
+    `changelog/TEMPLATE` for the content. It will be used in the
+    announcement of the next stable release. While writing, ask
+    yourself: If I were the user, what would I need to be aware of
+    with this change.
 
  8. Once your code looks good and passes all the tests, we'll merge it. Thanks
     a lot for your contribution!
@@ -141,13 +166,14 @@ run
 
     gofmt -w **/*.go
 
-in the project root directory before committing. Installing the script
-`fmt-check` from https://github.com/edsrzf/gofmt-git-hook locally as a
-pre-commit hook checks formatting before committing automatically, just copy
-this script to `.git/hooks/pre-commit`.
+in the project root directory before committing. For each Pull Request, the
+formatting is tested with `gofmt` for the latest stable version of Go.
+Installing the script `fmt-check` from https://github.com/edsrzf/gofmt-git-hook
+locally as a pre-commit hook checks formatting before committing automatically,
+just copy this script to `.git/hooks/pre-commit`.
 
 For each pull request, several different systems run the integration tests on
-Linux, OS X and Windows. We won't merge any code that does not pass all tests
+Linux, macOS and Windows. We won't merge any code that does not pass all tests
 for all systems, so when a tests fails, try to find out what's wrong and fix
 it. If you need help on this, please leave a comment in the pull request, and
 we'll be glad to assist. Having a PR with failing integration tests is nothing

Gopkg.lock

@@ -1,474 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  branch = "master"
-  digest = "1:94e9caf404409a2990cfd22aca37d758494c098eff3e2c37fda1abed862e74dd"
-  name = "bazil.org/fuse"
-  packages = [
-    ".",
-    "fs",
-    "fuseutil",
-  ]
-  pruneopts = "UT"
-  revision = "65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e"
-
-[[projects]]
-  digest = "1:5c3894b2aa4d6bead0ceeea6831b305d62879c871780e7b76296ded1b004bc57"
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata"]
-  pruneopts = "UT"
-  revision = "aad3f485ee528456e0768f20397b4d9dd941e755"
-  version = "v0.25.0"
-
-[[projects]]
-  digest = "1:46ea9487304f4b3c787f54483ecb13a338d686dcd670db0ab1a112ed0ae2128e"
-  name = "github.com/Azure/azure-sdk-for-go"
-  packages = [
-    "storage",
-    "version",
-  ]
-  pruneopts = "UT"
-  revision = "4e8cbbfb1aeab140cd0fa97fd16b64ee18c3ca6a"
-  version = "v19.1.0"
-
-[[projects]]
-  digest = "1:27d0cd1a78fc836f7c0f07749d029a5f7895c84ad066187b08b70e9d1830098e"
-  name = "github.com/Azure/go-autorest"
-  packages = [
-    "autorest",
-    "autorest/adal",
-    "autorest/azure",
-    "autorest/date",
-    "logger",
-    "version",
-  ]
-  pruneopts = "UT"
-  revision = "dd94e014aaf16d1df746762e392aa201c1b4c461"
-  version = "v10.15.0"
-
-[[projects]]
-  digest = "1:2209584c0f7c9b68c23374e659357ab546e1b70eec2761f03280f69a8fd23d77"
-  name = "github.com/cenkalti/backoff"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "2ea60e5f094469f9e65adb9cd103795b73ae743e"
-  version = "v2.0.0"
-
-[[projects]]
-  digest = "1:7cb4fdca4c251b3ef8027c90ea35f70c7b661a593b9eeae34753c65499098bb1"
-  name = "github.com/cpuguy83/go-md2man"
-  packages = ["md2man"]
-  pruneopts = "UT"
-  revision = "20f5889cbdc3c73dbd2862796665e7c465ade7d1"
-  version = "v1.0.8"
-
-[[projects]]
-  digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55"
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
-  version = "v3.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:6f9339c912bbdda81302633ad7e99a28dfa5a639c864061f1929510a9a64aa74"
-  name = "github.com/dustin/go-humanize"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "9f541cc9db5d55bce703bd99987c9d5cb8eea45e"
-
-[[projects]]
-  digest = "1:c7edfbb6320d6a93240d663dc52bca92bed4c116abe54c35679eec4e7cc2bd77"
-  name = "github.com/elithrar/simple-scrypt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "d150773194090feb6c897805a7bcea8d49544e2c"
-  version = "v1.3.0"
-
-[[projects]]
-  digest = "1:fe8a03a8222d5b913f256972933d26d24ad7c8286692a42943bc01633cc8fce3"
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
-  version = "v1.38.1"
-
-[[projects]]
-  digest = "1:15042ad3498153684d09f393bbaec6b216c8eec6d61f63dff711de7d64ed8861"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = "UT"
-  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:2e3c336fc7fde5c984d2841455a658a6d626450b1754a854b3b32e7a8f49a07a"
-  name = "github.com/google/go-cmp"
-  packages = [
-    "cmp",
-    "cmp/internal/diff",
-    "cmp/internal/function",
-    "cmp/internal/value",
-  ]
-  pruneopts = "UT"
-  revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
-  version = "v0.2.0"
-
-[[projects]]
-  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
-  name = "github.com/inconshreveable/mousetrap"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:190ff84d9b2ed6589088f178cba8edb4b8ecb334df4572421fb016be1ac20463"
-  name = "github.com/juju/ratelimit"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "59fac5042749a5afb9af70e813da1dd5474f0167"
-  version = "1.0.1"
-
-[[projects]]
-  digest = "1:9cedee824c21326bd26950bd9e1ffe9dc4e7ca03dc8634d0e6f954ee6a383172"
-  name = "github.com/kr/fs"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1455def202f6e05b95cc7bfc7e8ae67ae5141eba"
-  version = "v0.1.0"
-
-[[projects]]
-  digest = "1:1faa76bd9bffce9c25eaca0597afb67bd05a21ae57fe4378154ce8855ef163d1"
-  name = "github.com/kurin/blazer"
-  packages = [
-    "b2",
-    "base",
-    "internal/b2assets",
-    "internal/b2types",
-    "internal/blog",
-    "x/window",
-  ]
-  pruneopts = "UT"
-  revision = "caf65aa76491dc533bac68ad3243ce72fa4e0a0a"
-  version = "v0.5.1"
-
-[[projects]]
-  digest = "1:4e878df5f4e9fd625bf9c9aac77ef7cbfa4a74c01265505527c23470c0e40300"
-  name = "github.com/marstr/guid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "8bd9a64bf37eb297b492a4101fb28e80ac0b290f"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:d4d17353dbd05cb52a2a52b7fe1771883b682806f68db442b436294926bbfafb"
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:95c73c666919be2843b955eafc83f58c136312b74f79c703152f4c4a95fd64dc"
-  name = "github.com/minio/minio-go"
-  packages = [
-    ".",
-    "pkg/credentials",
-    "pkg/encrypt",
-    "pkg/s3signer",
-    "pkg/s3utils",
-    "pkg/set",
-  ]
-  pruneopts = "UT"
-  revision = "70799fe8dae6ecfb6c7d7e9e048fce27f23a1992"
-  version = "v6.0.5"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8eb17c2ec4df79193ae65b621cd1c0c4697db3bc317fe6afdc76d7f2746abd05"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66"
-
-[[projects]]
-  digest = "1:928de5172dd3563964d1b88a4ee3775cf72e16f1efabb482ab6d0e0bab86ee69"
-  name = "github.com/ncw/swift"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b2a7479cf26fa841ff90dd932d0221cb5c50782d"
-  version = "v1.0.39"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:cfa0d7741863a0e1d30e0ccdd4b48a96a471cdb47892303de8b92c3713af3e77"
-  name = "github.com/pkg/profile"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "5b67d428864e92711fcbd2f8629456121a56d91f"
-  version = "v1.2.1"
-
-[[projects]]
-  digest = "1:23ed92ba5d90a2dfe817f3895027ccef796e79c30be5125d48e17afdcc395d73"
-  name = "github.com/pkg/sftp"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "57673e38ea946592a59c26592b7e6fbda646975b"
-  version = "v1.8.0"
-
-[[projects]]
-  digest = "1:0d67664e93e366f072ac9672feea29bfc63c9f90f005e9e8a0df1954153f5a14"
-  name = "github.com/pkg/xattr"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ae385d07bb53f092fcc7daaf738d8513df084931"
-  version = "v0.3.1"
-
-[[projects]]
-  digest = "1:13ecc4000f49cf0aa3ee56fffcc93119c8edffacfff08674c80d2757d8c33a83"
-  name = "github.com/restic/chunker"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "db83917be3b88cc307464b7d8a221c173e34a0db"
-  version = "v0.2.0"
-
-[[projects]]
-  digest = "1:8bc629776d035c003c7814d4369521afe67fdb8efc4b5f66540d29343b98cf23"
-  name = "github.com/russross/blackfriday"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "55d61fa8aa702f59229e6cff85793c22e580eaf5"
-  version = "v1.5.1"
-
-[[projects]]
-  digest = "1:274f67cb6fed9588ea2521ecdac05a6d62a8c51c074c1fccc6a49a40ba80e925"
-  name = "github.com/satori/go.uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc"
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3e01752db0189b9157070a0e1668a620f9a85da2"
-  version = "v1.0.6"
-
-[[projects]]
-  digest = "1:e01b05ba901239c783dfe56450bcde607fc858908529868259c9a8765dc176d0"
-  name = "github.com/spf13/cobra"
-  packages = [
-    ".",
-    "doc",
-  ]
-  pruneopts = "UT"
-  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7"
-  name = "github.com/spf13/pflag"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:eefb1f49ec07e71206d4c9ea1a3e634cad331c2180733e4121b8ae39e8e92ecb"
-  name = "golang.org/x/crypto"
-  packages = [
-    "argon2",
-    "blake2b",
-    "curve25519",
-    "ed25519",
-    "ed25519/internal/edwards25519",
-    "internal/chacha20",
-    "internal/subtle",
-    "pbkdf2",
-    "poly1305",
-    "scrypt",
-    "ssh",
-    "ssh/terminal",
-  ]
-  pruneopts = "UT"
-  revision = "c126467f60eb25f8f27e5a981f32a87e3965053f"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8356aa7bdcb10a210b814b64ff76d61de7c36ac4cb6263de3af5e3e2e546956d"
-  name = "golang.org/x/net"
-  packages = [
-    "context",
-    "context/ctxhttp",
-    "http/httpguts",
-    "http2",
-    "http2/hpack",
-    "idna",
-  ]
-  pruneopts = "UT"
-  revision = "32f9bdbd7df18e8641d215e7ea68be88b971feb0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:bea0314c10bd362ab623af4880d853b5bad3b63d0ab9945c47e461b8d04203ed"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "google",
-    "internal",
-    "jws",
-    "jwt",
-  ]
-  pruneopts = "UT"
-  revision = "3d292e4d0cdc3a0113e6d207bb137145ef1de42f"
-
-[[projects]]
-  branch = "master"
-  digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  pruneopts = "UT"
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a220a85c72a6cb7339c412cb2b117019a7fd94007cdfffb3b5b1d058227a2bf8"
-  name = "golang.org/x/sys"
-  packages = [
-    "cpu",
-    "unix",
-    "windows",
-  ]
-  pruneopts = "UT"
-  revision = "bd9dbc187b6e1dacfdd2722a87e83093c2d7bd6e"
-
-[[projects]]
-  digest = "1:e5a8511f063c38c51ab9ab80e718e9149f692652aeb4e393a8c020dd1bf38ca2"
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "encoding",
-    "encoding/internal",
-    "encoding/internal/identifier",
-    "encoding/unicode",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "internal/utf8internal",
-    "language",
-    "runes",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable",
-  ]
-  pruneopts = "UT"
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:fb983acae7bd9c3ed9aadc1b1241d9e559ed21dbf84c17a0dda663ca169ccd69"
-  name = "google.golang.org/api"
-  packages = [
-    "gensupport",
-    "googleapi",
-    "googleapi/internal/uritemplates",
-    "storage/v1",
-  ]
-  pruneopts = "UT"
-  revision = "31ca0e01cd791f07750cb23fc99327721f753290"
-
-[[projects]]
-  digest = "1:c8907869850adaa8bd7631887948d0684f3787d0912f1c01ab72581a6c34432e"
-  name = "google.golang.org/appengine"
-  packages = [
-    ".",
-    "internal",
-    "internal/app_identity",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/modules",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:5bb148b78468350091db2ffbb2370f35cc6dcd74d9378a31b1c7b86ff7528f08"
-  name = "gopkg.in/tomb.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "d5d1b5820637886def9eef33e03a27a9f166942c"
-
-[[projects]]
-  digest = "1:342378ac4dcb378a5448dd723f0784ae519383532f5e70ade24132c4c8693202"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "bazil.org/fuse",
-    "bazil.org/fuse/fs",
-    "github.com/Azure/azure-sdk-for-go/storage",
-    "github.com/cenkalti/backoff",
-    "github.com/elithrar/simple-scrypt",
-    "github.com/google/go-cmp/cmp",
-    "github.com/juju/ratelimit",
-    "github.com/kurin/blazer/b2",
-    "github.com/mattn/go-isatty",
-    "github.com/minio/minio-go",
-    "github.com/minio/minio-go/pkg/credentials",
-    "github.com/ncw/swift",
-    "github.com/pkg/errors",
-    "github.com/pkg/profile",
-    "github.com/pkg/sftp",
-    "github.com/pkg/xattr",
-    "github.com/restic/chunker",
-    "github.com/spf13/cobra",
-    "github.com/spf13/cobra/doc",
-    "github.com/spf13/pflag",
-    "golang.org/x/crypto/poly1305",
-    "golang.org/x/crypto/scrypt",
-    "golang.org/x/crypto/ssh/terminal",
-    "golang.org/x/net/context",
-    "golang.org/x/net/context/ctxhttp",
-    "golang.org/x/net/http2",
-    "golang.org/x/oauth2/google",
-    "golang.org/x/sync/errgroup",
-    "golang.org/x/sys/unix",
-    "golang.org/x/text/encoding/unicode",
-    "google.golang.org/api/googleapi",
-    "google.golang.org/api/storage/v1",
-    "gopkg.in/tomb.v2",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,25 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-[prune]
-  unused-packages = true
-  go-tests = true

README.rst

@@ -1,9 +1,9 @@
-|Documentation| |Build Status| |Build status| |Report Card| |Say Thanks| |TestCoverage| |Reviewed by Hound|
+|Documentation| |Build Status| |Build status| |Report Card| |Say Thanks| |Reviewed by Hound|
 
 Introduction
 ------------
 
-restic is a backup program that is fast, efficient and secure.
+restic is a backup program that is fast, efficient and secure. It supports the three major operating systems (Linux, macOS, Windows) and a few smaller ones (FreeBSD, OpenBSD).
 
 For detailed usage and installation instructions check out the `documentation <https://restic.readthedocs.io/en/latest>`__.
 
@@ -129,8 +129,6 @@ Storage are sponsored by `AppsCode <https://appscode.com>`__!
    :target: https://goreportcard.com/report/github.com/restic/restic
 .. |Say Thanks| image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg
    :target: https://saythanks.io/to/restic
-.. |TestCoverage| image:: https://codecov.io/gh/restic/restic/branch/master/graph/badge.svg
-   :target: https://codecov.io/gh/restic/restic
 .. |AppsCode| image:: https://cdn.appscode.com/images/logo/appscode/ac-logo-color.png
    :target: https://appscode.com
 .. |Reviewed by Hound| image:: https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg

VERSION

@@ -1 +1 @@
-0.9.2
+0.9.6

appveyor.yml

@@ -7,6 +7,9 @@ branches:
   only:
     - master
 
+cache:
+  - '%LocalAppData%\go-build'
+
 init:
   - ps: >-
       $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
@@ -17,8 +20,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.10.windows-amd64.msi
-  - msiexec /i go1.10.windows-amd64.msi /q
+  - appveyor DownloadFile https://dl.google.com/go/go1.14.windows-amd64.msi
+  - msiexec /i go1.14.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

build.go

@@ -1,9 +1,17 @@
+// Description
+//
+// This program aims to make building Go programs for end users easier by just
+// calling it with `go run`, without having to setup a GOPATH.
+//
+// This program needs Go >= 1.11. It'll use Go modules for compilation. It
+// builds the package configured as Main in the Config struct.
+
 // BSD 2-Clause License
 //
 // Copyright (c) 2016-2018, Alexander Neumann <alexander@bumpern.de>
 // All rights reserved.
 //
-// This file has been copied from the repository at:
+// This file has been derived from the repository at:
 // https://github.com/fd0/build-go
 //
 // Redistribution and use in source and binary forms, with or without
@@ -37,7 +45,6 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -46,169 +53,32 @@ import (
 
 // config contains the configuration for the program to build.
 var config = Config{
-	Name:      "restic",                              // name of the program executable and directory
-	Namespace: "github.com/restic/restic",            // subdir of GOPATH, e.g. "github.com/foo/bar"
-	Main:      "github.com/restic/restic/cmd/restic", // package name for the main package
-	Tests: []string{ // tests to run
-		"github.com/restic/restic/internal/...",
-		"github.com/restic/restic/cmd/...",
-	},
-	MinVersion: GoVersion{Major: 1, Minor: 9, Patch: 0}, // minimum Go version supported
+	Name:             "restic",                                 // name of the program executable and directory
+	Namespace:        "github.com/restic/restic",               // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Main:             "./cmd/restic",                           // package name for the main package
+	DefaultBuildTags: []string{"selfupdate"},                   // specify build tags which are always used
+	Tests:            []string{"./..."},                        // tests to run
+	MinVersion:       GoVersion{Major: 1, Minor: 11, Patch: 0}, // minimum Go version supported
 }
 
 // Config configures the build.
 type Config struct {
-	Name       string
-	Namespace  string
-	Main       string
-	Tests      []string
-	MinVersion GoVersion
+	Name             string
+	Namespace        string
+	Main             string
+	DefaultBuildTags []string
+	Tests            []string
+	MinVersion       GoVersion
 }
 
 var (
-	verbose    bool
-	keepGopath bool
-	runTests   bool
-	enableCGO  bool
-	enablePIE  bool
+	verbose   bool
+	runTests  bool
+	enableCGO bool
+	enablePIE bool
+	goVersion = ParseGoVersion(runtime.Version())
 )
 
-// specialDir returns true if the file begins with a special character ('.' or '_').
-func specialDir(name string) bool {
-	if name == "." {
-		return false
-	}
-
-	base := filepath.Base(name)
-	if base == "vendor" || base[0] == '_' || base[0] == '.' {
-		return true
-	}
-
-	return false
-}
-
-// excludePath returns true if the file should not be copied to the new GOPATH.
-func excludePath(name string) bool {
-	ext := path.Ext(name)
-	if ext == ".go" || ext == ".s" || ext == ".h" {
-		return false
-	}
-
-	parentDir := filepath.Base(filepath.Dir(name))
-	if parentDir == "testdata" {
-		return false
-	}
-
-	return true
-}
-
-// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
-// to dst/prefix/, so calling
-//
-//   updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic")
-//
-// with "/home/u/restic" containing the file "foo.go" yields the following tree
-// at "/tmp/gopath":
-//
-//   /tmp/gopath
-//   └── src
-//       └── github.com
-//           └── restic
-//               └── restic
-//                   └── foo.go
-func updateGopath(dst, src, prefix string) error {
-	verbosePrintf("copy contents of %v to %v\n", src, filepath.Join(dst, prefix))
-	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
-		if name == src {
-			return err
-		}
-
-		if specialDir(name) {
-			if fi.IsDir() {
-				return filepath.SkipDir
-			}
-
-			return nil
-		}
-
-		if err != nil {
-			return err
-		}
-
-		if fi.IsDir() {
-			return nil
-		}
-
-		if excludePath(name) {
-			return nil
-		}
-
-		intermediatePath, err := filepath.Rel(src, name)
-		if err != nil {
-			return err
-		}
-
-		fileSrc := filepath.Join(src, intermediatePath)
-		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)
-
-		return copyFile(fileDst, fileSrc)
-	})
-}
-
-func directoryExists(dirname string) bool {
-	stat, err := os.Stat(dirname)
-	if err != nil && os.IsNotExist(err) {
-		return false
-	}
-
-	return stat.IsDir()
-}
-
-// copyFile creates dst from src, preserving file attributes and timestamps.
-func copyFile(dst, src string) error {
-	fi, err := os.Stat(src)
-	if err != nil {
-		return err
-	}
-
-	fsrc, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-
-	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
-		fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst))
-		return err
-	}
-
-	fdst, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-
-	if _, err = io.Copy(fdst, fsrc); err != nil {
-		return err
-	}
-
-	if err == nil {
-		err = fsrc.Close()
-	}
-
-	if err == nil {
-		err = fdst.Close()
-	}
-
-	if err == nil {
-		err = os.Chmod(dst, fi.Mode())
-	}
-
-	if err == nil {
-		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
-	}
-
-	return nil
-}
-
 // die prints the message with fmt.Fprintf() to stderr and exits with an error
 // code.
 func die(message string, args ...interface{}) {
@@ -222,7 +92,6 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "OPTIONS:\n")
 	fmt.Fprintf(output, "  -v     --verbose       output more messages\n")
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
-	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
 	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --enable-cgo    use CGO to link against libc\n")
@@ -230,7 +99,6 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarm value   set GOARM for cross-compilation\n")
-	fmt.Fprintf(output, "         --tempdir dir   use a specific directory for compilation\n")
 }
 
 func verbosePrintf(message string, args ...interface{}) {
@@ -241,64 +109,77 @@ func verbosePrintf(message string, args ...interface{}) {
 	fmt.Printf("build: "+message, args...)
 }
 
-// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if
-// present).
-func cleanEnv() (env []string) {
-	for _, v := range os.Environ() {
-		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
+// printEnv prints Go-relevant environment variables in a nice way using verbosePrintf.
+func printEnv(env []string) {
+	verbosePrintf("environment (GO*):\n")
+	for _, v := range env {
+		// ignore environment variables which do not start with GO*.
+		if !strings.HasPrefix(v, "GO") {
 			continue
 		}
-
-		env = append(env, v)
+		verbosePrintf("  %s\n", v)
 	}
-
-	return env
 }
 
 // build runs "go build args..." with GOPATH set to gopath.
-func build(cwd string, ver GoVersion, goos, goarch, goarm, gopath string, args ...string) error {
+func build(cwd string, env map[string]string, args ...string) error {
 	a := []string{"build"}
 
-	if ver.AtLeast(GoVersion{1, 10, 0}) {
-		verbosePrintf("Go version is at least 1.10, using new syntax for -gcflags\n")
-		// use new prefix
-		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", gopath))
-		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", gopath))
+	// try to remove all absolute paths from resulting binary
+	if goVersion.AtLeast(GoVersion{1, 13, 0}) {
+		// use the new flag introduced by Go 1.13
+		a = append(a, "-trimpath")
 	} else {
-		a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", gopath))
-		a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", gopath))
+		// otherwise try to trim as many paths as possible
+		a = append(a, "-asmflags", fmt.Sprintf("all=-trimpath=%s", cwd))
+		a = append(a, "-gcflags", fmt.Sprintf("all=-trimpath=%s", cwd))
 	}
+
 	if enablePIE {
 		a = append(a, "-buildmode=pie")
 	}
 
 	a = append(a, args...)
 	cmd := exec.Command("go", a...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
-	if goarm != "" {
-		cmd.Env = append(cmd.Env, "GOARM="+goarm)
+	cmd.Env = os.Environ()
+	for k, v := range env {
+		cmd.Env = append(cmd.Env, k+"="+v)
 	}
 	if !enableCGO {
 		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
 	}
 
+	printEnv(cmd.Env)
+
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	verbosePrintf("go %s\n", a)
+
+	verbosePrintf("chdir %q\n", cwd)
+	verbosePrintf("go %q\n", a)
 
 	return cmd.Run()
 }
 
 // test runs "go test args..." with GOPATH set to gopath.
-func test(cwd, gopath string, args ...string) error {
-	args = append([]string{"test"}, args...)
+func test(cwd string, env map[string]string, args ...string) error {
+	args = append([]string{"test", "-count", "1"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
+	cmd.Env = os.Environ()
+	for k, v := range env {
+		cmd.Env = append(cmd.Env, k+"="+v)
+	}
+	if !enableCGO {
+		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
+	}
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	verbosePrintf("go %s\n", args)
+
+	printEnv(cmd.Env)
+
+	verbosePrintf("chdir %q\n", cwd)
+	verbosePrintf("go %q\n", args)
 
 	return cmd.Run()
 }
@@ -446,22 +327,26 @@ func (v GoVersion) String() string {
 }
 
 func main() {
-	ver := ParseGoVersion(runtime.Version())
-	if !ver.AtLeast(config.MinVersion) {
-		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", ver, config.MinVersion)
+	if !goVersion.AtLeast(GoVersion{1, 11, 0}) {
+		die("Go version (%v) is too old, Go <= 1.11 does not support Go Modules\n", goVersion)
+	}
+
+	if !goVersion.AtLeast(config.MinVersion) {
+		fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion)
 		os.Exit(1)
 	}
 
-	buildTags := []string{}
+	buildTags := config.DefaultBuildTags
 
 	skipNext := false
 	params := os.Args[1:]
 
-	targetGOOS := runtime.GOOS
-	targetGOARCH := runtime.GOARCH
-	targetGOARM := ""
-
-	gopath := ""
+	env := map[string]string{
+		"GO111MODULE": "on", // make sure we build in Module mode
+		"GOOS":        runtime.GOOS,
+		"GOARCH":      runtime.GOARCH,
+		"GOARM":       "",
+	}
 
 	var outputFilename string
 
@@ -474,20 +359,15 @@ func main() {
 		switch arg {
 		case "-v", "--verbose":
 			verbose = true
-		case "-k", "--keep-gopath":
-			keepGopath = true
 		case "-t", "-tags", "--tags":
 			if i+1 >= len(params) {
 				die("-t given but no tag specified")
 			}
 			skipNext = true
-			buildTags = strings.Split(params[i+1], " ")
+			buildTags = append(buildTags, strings.Split(params[i+1], " ")...)
 		case "-o", "--output":
 			skipNext = true
 			outputFilename = params[i+1]
-		case "--tempdir":
-			skipNext = true
-			gopath = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--enable-cgo":
@@ -496,13 +376,13 @@ func main() {
 			enablePIE = true
 		case "--goos":
 			skipNext = true
-			targetGOOS = params[i+1]
+			env["GOOS"] = params[i+1]
 		case "--goarch":
 			skipNext = true
-			targetGOARCH = params[i+1]
+			env["GOARCH"] = params[i+1]
 		case "--goarm":
 			skipNext = true
-			targetGOARM = params[i+1]
+			env["GOARM"] = params[i+1]
 		case "-h":
 			showUsage(os.Stdout)
 			return
@@ -513,15 +393,14 @@ func main() {
 		}
 	}
 
-	verbosePrintf("detected Go version %v\n", ver)
-
-	if len(buildTags) == 0 {
-		verbosePrintf("adding build-tag release\n")
-		buildTags = []string{"release"}
-	}
+	verbosePrintf("detected Go version %v\n", goVersion)
 
+	preserveSymbols := false
 	for i := range buildTags {
 		buildTags[i] = strings.TrimSpace(buildTags[i])
+		if buildTags[i] == "debug" || buildTags[i] == "profile" {
+			preserveSymbols = true
+		}
 	}
 
 	verbosePrintf("build tags: %s\n", buildTags)
@@ -531,50 +410,16 @@ func main() {
 		die("Getwd(): %v\n", err)
 	}
 
-	if gopath == "" {
-		gopath, err = ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
-		if err != nil {
-			die("TempDir(): %v\n", err)
-		}
-	}
-
-	verbosePrintf("create GOPATH at %v\n", gopath)
-	if err = updateGopath(gopath, root, config.Namespace); err != nil {
-		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
-	}
-
-	vendor := filepath.Join(root, "vendor")
-	if directoryExists(vendor) {
-		if err = updateGopath(gopath, vendor, filepath.Join(config.Namespace, "vendor")); err != nil {
-			die("copying files from %v to %v failed: %v\n", root, gopath, err)
-		}
-	}
-
-	defer func() {
-		if !keepGopath {
-			verbosePrintf("remove %v\n", gopath)
-			if err = os.RemoveAll(gopath); err != nil {
-				die("remove GOPATH at %s failed: %v\n", err)
-			}
-		} else {
-			verbosePrintf("leaving temporary GOPATH at %v\n", gopath)
-		}
-	}()
-
 	if outputFilename == "" {
 		outputFilename = config.Name
-		if targetGOOS == "windows" {
+		if env["GOOS"] == "windows" {
 			outputFilename += ".exe"
 		}
 	}
 
-	cwd, err := os.Getwd()
-	if err != nil {
-		die("Getwd() returned %v\n", err)
-	}
 	output := outputFilename
 	if !filepath.IsAbs(output) {
-		output = filepath.Join(cwd, output)
+		output = filepath.Join(root, output)
 	}
 
 	version := getVersion()
@@ -582,16 +427,36 @@ func main() {
 	if version != "" {
 		constants["main.version"] = version
 	}
-	ldflags := "-s -w " + constants.LDFlags()
+	ldflags := constants.LDFlags()
+	if !preserveSymbols {
+		// Strip debug symbols.
+		ldflags = "-s -w " + ldflags
+	}
 	verbosePrintf("ldflags: %s\n", ldflags)
 
-	args := []string{
-		"-tags", strings.Join(buildTags, " "),
-		"-ldflags", ldflags,
-		"-o", output, config.Main,
+	var (
+		buildArgs []string
+		testArgs  []string
+	)
+
+	mainPackage := config.Main
+	if strings.HasPrefix(mainPackage, config.Namespace) {
+		mainPackage = strings.Replace(mainPackage, config.Namespace, "./", 1)
 	}
 
-	err = build(filepath.Join(gopath, "src"), ver, targetGOOS, targetGOARCH, targetGOARM, gopath, args...)
+	buildTarget := filepath.FromSlash(mainPackage)
+	buildCWD, err := os.Getwd()
+	if err != nil {
+		die("unable to determine current working directory: %v\n", err)
+	}
+
+	buildArgs = append(buildArgs,
+		"-tags", strings.Join(buildTags, " "),
+		"-ldflags", ldflags,
+		"-o", output, buildTarget,
+	)
+
+	err = build(buildCWD, env, buildArgs...)
 	if err != nil {
 		die("build failed: %v\n", err)
 	}
@@ -599,7 +464,9 @@ func main() {
 	if runTests {
 		verbosePrintf("running tests\n")
 
-		err = test(cwd, gopath, config.Tests...)
+		testArgs = append(testArgs, config.Tests...)
+
+		err = test(buildCWD, env, testArgs...)
 		if err != nil {
 			die("running tests failed: %v\n", err)
 		}

changelog/0.9.3_2018-10-13/issue-1766

@@ -0,0 +1,7 @@
+Enhancement: restore: suppress lchown errors when not running as root
+
+Like "cp" and "rsync" do, restic now only reports errors for changing
+the ownership of files during restore if it is run as root, on non-Windows
+operating systems. On Windows, the error is reported as usual.
+
+https://github.com/restic/restic/issues/1766

changelog/0.9.3_2018-10-13/issue-1909

@@ -0,0 +1,14 @@
+Enhancement: Reject files/dirs by name first
+
+The current scanner/archiver code had an architectural limitation: it always
+ran the `lstat()` system call on all files and directories before a decision to
+include/exclude the file/dir was made. This lead to a lot of unnecessary system
+calls for items that could have been rejected by their name or path only.
+
+We've changed the archiver/scanner implementation so that it now first rejects
+by name/path, and only runs the system call on the remaining items. This
+reduces the number of `lstat()` system calls a lot (depending on the exclude
+settings).
+
+https://github.com/restic/restic/issues/1909
+https://github.com/restic/restic/pull/1912

changelog/0.9.3_2018-10-13/issue-1935

@@ -0,0 +1,8 @@
+Bugfix: Remove truncated files from cache
+
+When a file in the local cache is truncated, and restic tries to access data
+beyond the end of the (cached) file, it used to return an error "EOF". This is
+now fixed, such truncated files are removed and the data is fetched directly
+from the backend.
+
+https://github.com/restic/restic/issues/1935

changelog/0.9.3_2018-10-13/issue-1941

@@ -0,0 +1,15 @@
+Enhancement: Add directory filter to ls command
+
+The ls command can now be filtered by directories, so that only files in the
+given directories will be shown. If the --recursive flag is specified, then
+ls will traverse subfolders and list their files as well.
+
+It used to be possible to specify multiple snapshots, but that has been
+replaced by only one snapshot and the possibility of specifying multiple
+directories.
+
+Specifying directories constrains the walk, which can significantly speed up
+the listing.
+
+https://github.com/restic/restic/issues/1940
+https://github.com/restic/restic/pull/1941

changelog/0.9.3_2018-10-13/issue-1967

@@ -0,0 +1,7 @@
+Enhancement: Use `--host` everywhere
+
+We now use the flag `--host` for all commands which need a host name, using
+`--hostname` (e.g. for `restic backup`) still works, but will print a
+deprecation warning. Also, add the short option `-H` where possible.
+
+https://github.com/restic/restic/issues/1967

changelog/0.9.3_2018-10-13/issue-1978

@@ -0,0 +1,12 @@
+Bugfix: Do not return an error when the scanner is slower than backup
+
+When restic makes a backup, there's a background task called "scanner" which
+collects information on how many files and directories are to be saved, in
+order to display progress information to the user. When the backup finishes
+faster than the scanner, it is aborted because the result is not needed any
+more. This logic contained a bug, where quitting the scanner process was
+treated as an error, and caused restic to print an unhelpful error message
+("context canceled").
+
+https://github.com/restic/restic/issues/1978
+https://github.com/restic/restic/pull/1991

changelog/0.9.3_2018-10-13/issue-2028

@@ -0,0 +1,7 @@
+Enhancement: Display size of cache directories
+
+The `cache` command now by default shows the size of the individual cache
+directories. It can be disabled with `--no-size`.
+
+https://github.com/restic/restic/issues/2028
+https://github.com/restic/restic/pull/2033

changelog/0.9.3_2018-10-13/pull-1780

@@ -0,0 +1,13 @@
+Enhancement: Improve the `find` command
+
+We've updated the `find` command to support multiple patterns.
+
+`restic find` is now able to list the snapshots containing a specific tree
+or blob, or even the snapshots that contain blobs belonging to a given pack.
+A list of IDs can be given, as long as they all have the same type.
+
+The command `find` can also display the pack IDs the blobs belong to, if
+the `--show-pack-id` flag is provided.
+
+https://github.com/restic/restic/issues/1777
+https://github.com/restic/restic/pull/1780

changelog/0.9.3_2018-10-13/pull-1876

@@ -0,0 +1,7 @@
+Enhancement: Display reason why forget keeps snapshots
+
+We've added a column to the list of snapshots `forget` keeps which details the
+reasons to keep a particuliar snapshot. This makes debugging policies for
+forget much easier. Please remember to always try things out with `--dry-run`!
+
+https://github.com/restic/restic/pull/1876

changelog/0.9.3_2018-10-13/pull-1891

@@ -0,0 +1,7 @@
+Enhancement: Accept glob in paths loaded via --files-from
+
+Before that, behaviour was different if paths were appended to command line or
+from a file, because wild card characters were expanded by shell if appended to
+command line, but not expanded if loaded from file.
+
+https://github.com/restic/restic/issues/1891

changelog/0.9.3_2018-10-13/pull-1920

@@ -0,0 +1,8 @@
+Enhancement: Vendor dependencies with Go 1.11 Modules
+
+Until now, we've used `dep` for managing dependencies, we've now switch to
+using Go modules. For users this does not change much, only if you want to
+compile restic without downloading anything with Go 1.11, then you need to run:
+`go build -mod=vendor build.go`
+
+https://github.com/restic/restic/pull/1920

changelog/0.9.3_2018-10-13/pull-1949

@@ -0,0 +1,15 @@
+Enhancement: Add new command `self-update`
+
+We have added a new command called `self-update` which downloads the
+latest released version of restic from GitHub and replaces the current
+binary with it. It does not rely on any external program (so it'll work
+everywhere), but still verifies the GPG signature using the embedded GPG
+public key.
+
+By default, the `self-update` command is hidden behind the `selfupdate`
+built tag, which is only set when restic is built using `build.go` (including
+official releases). The reason for this is that downstream distributions will
+then not include the command by default, so users are encouraged to use the
+platform-specific distribution mechanism.
+
+https://github.com/restic/restic/pull/1949

changelog/0.9.3_2018-10-13/pull-1953

@@ -0,0 +1,7 @@
+Enhancement: ls: Add JSON output support for restic ls cmd
+
+We've implemented listing files in the repository with JSON as output, just
+pass `--json` as an option to `restic ls`. This makes the output of the command
+machine readable.
+
+https://github.com/restic/restic/pull/1953

changelog/0.9.3_2018-10-13/pull-1962

@@ -0,0 +1,13 @@
+Enhancement: Stream JSON output for ls command
+
+The `ls` command now supports JSON output with the global `--json`
+flag, and this change streams out JSON messages one object at a time
+rather than en entire array buffered in memory before encoding. The
+advantage is it allows large listings to be handled efficiently.
+
+Two message types are printed: snapshots and nodes. A snapshot
+object will precede node objects which belong to that snapshot.
+The `struct_type` field can be used to determine which kind of
+message an object is.
+
+https://github.com/restic/restic/pull/1962

changelog/0.9.4_2019-01-06/issue-1605

@@ -0,0 +1,11 @@
+Enhancement: Concurrent restore
+
+This change significantly improves restore performance, especially
+when using high-latency remote repositories like B2.
+
+The implementation now uses several concurrent threads to download and process
+multiple remote files concurrently. To further reduce restore time, each remote
+file is downloaded using a single repository request.
+
+https://github.com/restic/restic/issues/1605
+https://github.com/restic/restic/pull/1719

changelog/0.9.4_2019-01-06/issue-1989

@@ -0,0 +1,7 @@
+Bugfix: Google Cloud Storage: Respect bandwidth limit
+
+The GCS backend did not respect the bandwidth limit configured, a previous
+commit accidentally removed support for it.
+
+https://github.com/restic/restic/issues/1989
+https://github.com/restic/restic/pull/2100

changelog/0.9.4_2019-01-06/issue-2040

@@ -0,0 +1,11 @@
+Bugfix: Add host name filter shorthand flag for `stats` command
+
+The default value for `--host` flag was set to 'H' (the shorthand version of
+the flag), this caused the lookup for the latest snapshot to fail.
+
+Add shorthand flag `-H` for `--host` (with empty default so if these flags
+are not specified the latest snapshot will not filter by host name).
+
+Also add shorthand `-H` for `backup` command.
+
+https://github.com/restic/restic/issues/2040

changelog/0.9.4_2019-01-06/issue-2089

@@ -0,0 +1,9 @@
+Enhancement: increase granularity of the "keep within" retention policy
+
+The `keep-within` option of the `forget` command now accepts time ranges with
+an hourly granularity. For example, running `restic forget --keep-within 3d12h`
+will keep all the snapshots made within three days and twelve hours from the
+time of the latest snapshot.
+
+https://github.com/restic/restic/issues/2089
+https://github.com/restic/restic/pull/2090

changelog/0.9.4_2019-01-06/issue-2097

@@ -0,0 +1,12 @@
+Enhancement: Add key hinting
+
+Added a new option `--key-hint` and corresponding environment variable
+`RESTIC_KEY_HINT`.  The key hint is a key ID to try decrypting first, before
+other keys in the repository.
+
+This change will benefit repositories with many keys; if the correct key hint
+is supplied then restic only needs to check one key.  If the key hint is
+incorrect (the key does not exist, or the password is incorrect) then restic
+will check all keys, as usual.
+
+https://github.com/restic/restic/issues/2097

changelog/0.9.4_2019-01-06/pull-2017

@@ -0,0 +1,11 @@
+Enhancement: mount: Enforce FUSE Unix permissions with allow-other
+
+The fuse mount (`restic mount`) now lets the kernel check the permissions of
+the files within snapshots (this is done through the `DefaultPermissions` FUSE
+option) when the option `--allow-other` is specified.
+
+To restore the old behavior, we've added the `--no-default-permissions` option.
+This allows all users that have access to the mount point to access all
+files within the snapshots.
+
+https://github.com/restic/restic/pull/2017

changelog/0.9.4_2019-01-06/pull-2068

@@ -0,0 +1,6 @@
+Bugfix: Correctly return error loading data
+
+In one case during `prune` and `check`, an error loading data from the backend is not returned properly. This is now corrected.
+
+https://github.com/restic/restic/pull/2068
+https://github.com/restic/restic/issues/1999#issuecomment-433737921

changelog/0.9.4_2019-01-06/pull-2070

@@ -0,0 +1,7 @@
+Enhancement: Make all commands display timestamps in local time
+
+Restic used to drop the timezone information from displayed timestamps, it now
+converts timestamps to local time before printing them so the times can be
+easily compared to.
+
+https://github.com/restic/restic/pull/2070

changelog/0.9.4_2019-01-06/pull-2086

@@ -0,0 +1,7 @@
+Enhancement: Allow --files-from to be specified multiple times
+
+Before, restic took only the last file specified with `--files-from` into
+account, this is now corrected.
+
+https://github.com/restic/restic/issues/2085
+https://github.com/restic/restic/pull/2086

changelog/0.9.4_2019-01-06/pull-2094

@@ -0,0 +1,8 @@
+Enhancement: Run command to get password
+
+We've added the `--password-command` option which allows specifying a command
+that restic runs every time the password for the repository is needed, so it
+can be integrated with a password manager or keyring. The option can also be
+set via the environment variable `$RESTIC_PASSWORD_COMMAND`.
+
+https://github.com/restic/restic/pull/2094

changelog/0.9.4_2019-01-06/pull-2095

@@ -0,0 +1,7 @@
+Bugfix: consistently use local time for snapshots times
+
+By default snapshots created with restic backup were set to local time,
+but when the --time flag was used the provided timestamp was parsed as
+UTC. With this change all snapshots times are set to local time.
+
+https://github.com/restic/restic/pull/2095

changelog/0.9.5_2019-04-23/issue-1895

@@ -0,0 +1,7 @@
+Enhancement: Add case insensitive include & exclude options
+
+The backup and restore commands now have --iexclude and --iinclude flags
+as case insensitive variants of --exclude and --include.
+
+https://github.com/restic/restic/issues/1895
+https://github.com/restic/restic/pull/2032

changelog/0.9.5_2019-04-23/issue-1937

@@ -0,0 +1,8 @@
+Enhancement: Support streaming JSON output for backup
+
+We've added support for getting machine-readable status output during backup,
+just pass the flag `--json` for `restic backup` and restic will output a stream
+of JSON objects which contain the current progress.
+
+https://github.com/restic/restic/issues/1937
+https://github.com/restic/restic/pull/1944

changelog/0.9.5_2019-04-23/issue-2135

@@ -0,0 +1,10 @@
+Bugfix: Return error when no bytes could be read from stdin
+
+We assume that users reading backup data from stdin want to know when no data
+could be read, so now restic returns an error when `backup --stdin` is called
+but no bytes could be read. Usually, this means that an earlier command in a
+pipe has failed. The documentation was amended and now recommends setting the
+`pipefail` option (`set -o pipefail`).
+
+https://github.com/restic/restic/pull/2135
+https://github.com/restic/restic/pull/2139

changelog/0.9.5_2019-04-23/issue-2155

@@ -0,0 +1,8 @@
+Enhancement: add Openstack application credential auth for Swift
+
+Since Openstack Queens Identity (auth V3) service supports an application
+credential auth method.  It allows to create a technical account with the
+limited roles. This commit adds an application credential authentication
+method for the Swift backend.
+
+https://github.com/restic/restic/issues/2155

changelog/0.9.5_2019-04-23/issue-2181

@@ -0,0 +1,3 @@
+Bugfix: Don't cancel timeout after 30 seconds for self-update
+
+https://github.com/restic/restic/issues/2181

changelog/0.9.5_2019-04-23/issue-2184

@@ -0,0 +1,8 @@
+Enhancement: Add --json support to forget command
+
+The forget command now supports the --json argument, outputting the
+information about what is (or would-be) kept and removed from the
+repository.
+
+https://github.com/restic/restic/issues/2184
+https://github.com/restic/restic/pull/2185

changelog/0.9.5_2019-04-23/issue-2203

@@ -0,0 +1,6 @@
+Bugfix: Fix reading passwords from stdin
+
+Passwords for the `init`, `key add`, and `key passwd` commands can now be read from
+non-terminal stdin.
+
+https://github.com/restic/restic/issues/2203

changelog/0.9.5_2019-04-23/issue-2224

@@ -0,0 +1,9 @@
+Bugfix: Don't abort the find command when a tree can't be loaded
+
+Change the find command so that missing trees don't result in a crash.
+Instead, the error is logged to the debug log, and the tree ID is displayed
+along with the snapshot it belongs to.  This makes it possible to recover
+repositories that are missing trees by forgetting the snapshots they are used
+in.
+
+https://github.com/restic/restic/issues/2224

changelog/0.9.5_2019-04-23/pull-2087

@@ -0,0 +1,10 @@
+Enhancement: Add group-by option to snapshots command
+
+We have added an option to group the output of the snapshots command, similar
+to the output of the forget command. The option has been called "--group-by"
+and accepts any combination of the values "host", "paths" and "tags", separated
+by commas. Default behavior (not specifying --group-by) has not been changed.
+We have added support of the grouping to the JSON output.
+
+https://github.com/restic/restic/issues/2037
+https://github.com/restic/restic/pull/2087

changelog/0.9.5_2019-04-23/pull-2124

@@ -0,0 +1,8 @@
+Enhancement: Ability to dump folders to tar via stdout
+
+We've added the ability to dump whole folders to stdout via the `dump` command.
+Restic now requires at least Go 1.10 due to a limitation of the standard
+library for Go <= 1.9.
+
+https://github.com/restic/restic/pull/2124
+https://github.com/restic/restic/issues/2123

changelog/0.9.5_2019-04-23/pull-2139

@@ -0,0 +1,8 @@
+Enhancement: Return error if no bytes could be read for `backup --stdin`
+
+When restic is used to backup the output of a program, like `mysqldump | restic
+backup --stdin`, it now returns an error if no bytes could be read at all. This
+catches the failure case when `mysqldump` failed for some reason and did not
+output any data to stdout.
+
+https://github.com/restic/restic/pull/2139

changelog/0.9.5_2019-04-23/pull-2205

@@ -0,0 +1,10 @@
+Enhancement: Add --ignore-inode option to backup cmd
+
+This option handles backup of virtual filesystems that do not keep fixed
+inodes for files, like Fuse-based, pCloud, etc. Ignoring inode changes allows
+to consider the file as unchanged if last modification date and size
+are unchanged.
+
+https://github.com/restic/restic/pull/2205
+https://github.com/restic/restic/pull/2047
+https://github.com/restic/restic/issues/1631

changelog/0.9.5_2019-04-23/pull-2220

@@ -0,0 +1,16 @@
+Enhancement: Add config option to set S3 storage class
+
+The `s3.storage-class` option can be passed to restic (using `-o`) to
+specify the storage class to be used for S3 objects created by restic.
+
+The storage class is passed as-is to S3, so it needs to be understood by
+the API. On AWS, it can be one of `STANDARD`, `STANDARD_IA`,
+`ONEZONE_IA`, `INTELLIGENT_TIERING` and `REDUCED_REDUNDANCY`. If
+unspecified, the default storage class is used (`STANDARD` on AWS).
+
+You can mix storage classes in the same bucket, and the setting isn't
+stored in the restic repository, so be sure to specify it with each
+command that writes to S3.
+
+https://github.com/restic/restic/pull/2220
+https://github.com/restic/restic/issues/706

changelog/0.9.6_2019-11-22/issue-2063

@@ -0,0 +1,6 @@
+Bugfix: Allow absolute path for filename when backing up from stdin
+
+When backing up from stdin, handle directory path for `--stdin-filename`.
+This can be used to specify the full path for the backed-up file.
+
+https://github.com/restic/restic/issues/2063

changelog/0.9.6_2019-11-22/issue-2174

@@ -0,0 +1,10 @@
+Bugfix: Save files with invalid timestamps
+
+When restic reads invalid timestamps (year is before 0000 or after 9999) it
+refused to read and archive the file. We've changed the behavior and will now
+save modified timestamps with the year set to either 0000 or 9999, the rest of
+the timestamp stays the same, so the file will be saved (albeit with a bogus
+timestamp).
+
+https://github.com/restic/restic/issues/2174
+https://github.com/restic/restic/issues/1173

changelog/0.9.6_2019-11-22/issue-2179

@@ -0,0 +1,15 @@
+Enhancement: Use ctime when checking for file changes
+
+Previously, restic only checked a file's mtime (along with other non-timestamp
+metadata) to decide if a file has changed. This could cause restic to not notice
+that a file has changed (and therefore continue to store the old version, as
+opposed to the modified version) if something edits the file and then resets the
+timestamp. Restic now also checks the ctime of files, so any modifications to a
+file should be noticed, and the modified file will be backed up. The ctime check
+will be disabled if the --ignore-inode flag was given.
+
+If this change causes problems for you, please open an issue, and we can look in
+to adding a seperate flag to disable just the ctime check.
+
+https://github.com/restic/restic/issues/2179
+https://github.com/restic/restic/pull/2212

changelog/0.9.6_2019-11-22/issue-2249

@@ -0,0 +1,6 @@
+Bugfix: Read fresh metadata for unmodified files
+
+Restic took all metadata for files which were detected as unmodified, not taking into account changed metadata (ownership, mode). This is now corrected.
+
+https://github.com/restic/restic/issues/2249
+https://github.com/restic/restic/pull/2252

changelog/0.9.6_2019-11-22/issue-2301

@@ -0,0 +1,7 @@
+Bugfix: Add upper bound for t in --read-data-subset=n/t
+
+256 is the effective maximum for t, but restic would allow larger
+values, leading to strange behavior.
+
+https://github.com/restic/restic/issues/2301
+https://github.com/restic/restic/pull/2304

changelog/0.9.6_2019-11-22/issue-2306

@@ -0,0 +1,7 @@
+Enhancement: Allow multiple retries for interactive password input
+
+Restic used to quit if the repository password was typed incorrectly once.
+Restic will now ask the user again for the repository password if typed incorrectly.
+The user will now get three tries to input the correct password before restic quits. 
+
+https://github.com/restic/restic/issues/2306

changelog/0.9.6_2019-11-22/issue-2330

@@ -0,0 +1,6 @@
+Enhancement: Make `--group-by` accept both singular and plural
+
+One can now use the values `host`/`hosts`, `path`/`paths` and
+`tag` / `tags` interchangeably in the `--group-by` argument.
+
+https://github.com/restic/restic/issues/2330

changelog/0.9.6_2019-11-22/pull-2321

@@ -0,0 +1,8 @@
+Bugfix: Check errors when loading index files
+
+Restic now checks and handles errors which occur when loading index files, the
+missing check leads to odd errors (and a stack trace printed to users) later.
+This was reported in the forum.
+
+https://github.com/restic/restic/pull/2321
+https://forum.restic.net/t/check-rebuild-index-prune/1848/13

changelog/0.9.6_2019-11-22/pull-2350

@@ -0,0 +1,8 @@
+Enhancement: Add option to configure S3 region
+
+We've added a new option for setting the region when accessing an S3-compatible
+service. For some providers, it is required to set this to a valid value. You
+can do that either by setting the environment variable `AWS_DEFAULT_REGION` or
+using the option `s3.region`, e.g. like this: `-o s3.region="us-east-1"`.
+
+https://github.com/restic/restic/pull/2350

changelog/CHANGELOG.tmpl

@@ -16,7 +16,7 @@ Details
 {{ range $entry := .Entries }}{{ with $entry }}
  * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }}
 {{ range $par := .Paragraphs }}
-   {{ wrap $par 80 3 }}
+   {{ wrapIndent $par 80 3 }}
 {{ end -}}
 {{ range $url := .IssueURLs }}
    {{ $url -}}

changelog/TEMPLATE

@@ -9,4 +9,4 @@ in case there aren't any issue links) is used as the primary ID.
 
 https://github.com/restic/restic/issues/1234
 https://github.com/restic/restic/pull/55555
-https://forum.restic/.net/foo/bar/baz
+https://forum.restic.net/foo/bar/baz

changelog/unreleased/issue-1570

@@ -0,0 +1,9 @@
+Enhancement: Support specifying multiple host flags for various commands
+
+Previously commands didn't take more than one `--host` or `-H` argument into account, which could be limiting with e.g.
+the `forget` command.
+
+The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and `tag` commands will now take into account
+multiple `--host` and `-H` flags.
+
+https://github.com/restic/restic/issues/1570

changelog/unreleased/issue-2072

@@ -0,0 +1,5 @@
+Enhancement: Display snapshot date when using `restic find`
+
+Added the respective snapshot date to the output of `restic find`.
+
+https://github.com/restic/restic/issues/2072

changelog/unreleased/issue-2277

@@ -0,0 +1,5 @@
+Enhancement: Add support for ppc64le
+
+Adds support for ppc64le, the processor architecture from IBM.
+
+https://github.com/restic/restic/issues/2277

changelog/unreleased/issue-2281

@@ -0,0 +1,7 @@
+Bugfix: Handle format verbs like '%' properly in `find` output
+
+The JSON or "normal" output of the `find` command can now deal with file names
+that contain substrings which the Golang `fmt` package considers "format verbs"
+like `%s`.
+
+https://github.com/restic/restic/issues/2281

changelog/unreleased/issue-2298

@@ -0,0 +1,8 @@
+Bugfix: Do not hang when run as a background job
+
+Restic did hang on exit while restoring the terminal configuration when it was
+started as a background job, for example using `restic ... &`. This has been
+fixed by only restoring the terminal configuration when restic is interrupted
+while reading a password from the terminal.
+
+https://github.com/restic/restic/issues/2298

changelog/unreleased/issue-2389

@@ -0,0 +1,8 @@
+Bugfix: Fix mangled json output of backup command
+
+We've fixed a race condition in the json output of the backup command
+that could cause multiple lines to get mixed up. We've also ensured that
+the backup summary is printed last.
+
+https://github.com/restic/restic/issues/2389
+https://github.com/restic/restic/pull/2545

changelog/unreleased/issue-2390

@@ -0,0 +1,5 @@
+Bugfix: Refresh lock timestamp
+
+Long-running operations did not refresh lock timestamp, resulting in locks becoming stale. This is now fixed.
+
+https://github.com/restic/restic/issues/2390

changelog/unreleased/issue-2429

@@ -0,0 +1,6 @@
+Bugfix: backup --json reports total_bytes_processed as 0
+
+We've fixed the json output of total_bytes_processed. The non-json output
+was already fixed with pull request #2138 but left the json output untouched.
+
+https://github.com/restic/restic/issues/2429

changelog/unreleased/issue-2469

@@ -0,0 +1,5 @@
+Bugfix: Fix incorrect bytes stats in `diff` command
+
+In some cases, the wrong number of bytes (e.g. 16777215.998 TiB) were reported by the `diff` command. This is now fixed.
+
+https://github.com/restic/restic/issues/2469

changelog/unreleased/issue-2482

@@ -0,0 +1,9 @@
+Change: Remove vendored dependencies
+
+We've removed the vendored dependencies (in the subdir `vendor/`). When
+building restic, the Go compiler automatically fetches the dependencies. It
+will also cryptographically verify that the correct code has been fetched by
+using the hashes in `go.sum` (see the link to the documentation below).
+
+https://github.com/restic/restic/issues/2482
+https://golang.org/cmd/go/#hdr-Module_downloading_and_verification

changelog/unreleased/issue-2518

@@ -0,0 +1,16 @@
+Bugfix: Do not crash with Synology NAS sftp server
+
+It was found that when restic is used to store data on an sftp server on a
+Synology NAS with a relative path (one which does not start with a slash), it
+may go into an endless loop trying to create directories on the server. We've
+fixed this bug by using a function in the sftp library instead of our own
+implementation.
+
+The bug was discovered because the Synology sftp server behaves erratic with
+non-absolute path (e.g. `home/restic-repo`). This can be resolved by just using
+an absolute path instead (`/home/restic-repo`). We've also added a paragraph in
+the FAQ.
+
+https://github.com/restic/restic/issues/2518
+https://github.com/restic/restic/issues/2363
+https://github.com/restic/restic/pull/2530

changelog/unreleased/issue-2531

@@ -0,0 +1,5 @@
+Bugfix: Fix incorrect size calculation in `stats --mode restore-size` 
+
+The restore-size mode of stats was counting hard-linked files as if they were independent.
+
+https://github.com/restic/restic/issues/2531

changelog/unreleased/issue-2537

@@ -0,0 +1,5 @@
+Bugfix: Fix incorrect file counts in `stats --mode restore-size` 
+
+The restore-size mode of stats was failing to count empty directories and some files with hard links.
+
+https://github.com/restic/restic/issues/2537

changelog/unreleased/pull-2195

@@ -0,0 +1,17 @@
+Enhancement: Simplify and improve restore performance
+
+Significantly improves restore performance of large files (i.e. 50M+):
+https://github.com/restic/restic/issues/2074
+https://forum.restic.net/t/restore-using-rclone-gdrive-backend-is-slow/1112/8
+https://forum.restic.net/t/degraded-restore-performance-s3-backend/1400
+
+Fixes "not enough cache capacity" error during restore:
+https://github.com/restic/restic/issues/2244
+
+NOTE: This new implementation does not guarantee order in which blobs
+are written to the target files and, for example, the last blob of a
+file can be written to the file before any of the preceeding file blobs.
+It is therefore possible to have gaps in the data written to the target
+files if restore fails or interrupted by the user.
+
+https://github.com/restic/restic/pull/2195

changelog/unreleased/pull-2423

@@ -0,0 +1,5 @@
+Enhancement: support user@domain parsing as user
+
+Added the ability for user@domain-like users to be authenticated over SFTP servers.
+
+https://github.com/restic/restic/pull/2423

changelog/unreleased/pull-2576

@@ -0,0 +1,7 @@
+Enhancement: Improve the chunking algorithm
+
+We've updated the chunker library responsible for splitting files into smaller
+blocks. It should improve the chunking throughput by 5-10% depending on the
+CPU.
+
+https://github.com/restic/restic/pull/2576

changelog/unreleased/pull-2592

@@ -0,0 +1,6 @@
+Bugfix: SFTP backend supports IPv6 addresses
+
+The SFTP backend now supports IPv6 addresses natively, without relying on
+aliases in the external SSH configuration.
+
+https://github.com/restic/restic/pull/2592

changelog/unreleased/pull-2600

@@ -0,0 +1,6 @@
+Change: Require Go >= 1.11
+
+Restic now requires Go to be at least 1.11. This allows simplifications in the
+build process and removing workarounds.
+
+https://github.com/restic/restic/pull/2600

changelog/unreleased/pull-2607

@@ -0,0 +1,7 @@
+Bugfix: Honor RESTIC_CACHE_DIR environment variable on Mac and Windows
+
+On Mac and Windows, the RESTIC_CACHE_DIR environment variable was ignored.
+This variable can now be used on all platforms to set the directory where
+restic stores caches.
+
+https://github.com/restic/restic/pull/2607

changelog/unreleased/pull-2668

@@ -0,0 +1,6 @@
+Bugfix: Don't abort the stats command when data blobs are missing
+
+Runing the stats command in the blobs-per-file mode on a repository with
+missing data blobs previously resulted in a crash.
+
+https://github.com/restic/restic/pull/2668

chunker/.travis.yml

@@ -1,18 +1,20 @@
 language: go
 sudo: false
 
-go:
-  - 1.6.4
-  - 1.7.4
-  - tip
-
-os:
-  - linux
-  - osx
+matrix:
+  include:
+    - os: linux
+      go: "1.9.x"
+    - os: linux
+      go: "1.10.x"
+    - os: linux
+      go: "tip"
+    - os: osx
+      go: "1.10.x"
 
 install:
   - go get -t ./...
-  - go get -u github.com/golang/lint/golint
+  - go get -u golang.org/x/lint/golint
   - go get -u golang.org/x/tools/cmd/goimports
 
 script:

chunker/README.md

@@ -1,5 +1,5 @@
 [![GoDoc](https://godoc.org/github.com/restic/chunker?status.svg)](http://godoc.org/github.com/restic/chunker)
-[![Build Status](https://travis-ci.org/restic/chunker.svg?branch=master)](https://travis-ci.org/restic/chunker)
+[![Build Status](https://travis-ci.com/restic/chunker.svg?branch=master)](https://travis-ci.com/restic/chunker)
 
 The package `chunker` implements content-defined-chunking (CDC) based on a
 rolling Rabin Hash. The library is part of the [restic backup

chunker/chunker.go

@@ -2,6 +2,7 @@ package chunker
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"sync"
 )
@@ -47,7 +48,7 @@ type Chunk struct {
 
 type chunkerState struct {
 	window [windowSize]byte
-	wpos   int
+	wpos   uint
 
 	buf  []byte
 	bpos uint
@@ -225,6 +226,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 	tabout := c.tables.out
 	tabmod := c.tables.mod
 	polShift := c.polShift
+	// go guarantees the expected behavior for bit shifts even for shift counts
+	// larger than the value width. Bounding the value of polShift allows the compiler
+	// to optimize the code for 'digest >> polShift'
+	if polShift > 53-8 {
+		return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
+	}
 	minSize := c.MinSize
 	maxSize := c.MaxSize
 	buf := c.buf
@@ -259,6 +266,10 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 				return Chunk{}, err
 			}
 
+			if n < 0 {
+				return Chunk{}, fmt.Errorf("ReadFull returned negative number of bytes read: %v", n)
+			}
+
 			c.bpos = 0
 			c.bmax = uint(n)
 		}
@@ -291,10 +302,12 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 		wpos := c.wpos
 		for _, b := range buf[c.bpos:c.bmax] {
 			// slide(b)
+			// limit wpos before to elide array bound checks
+			wpos = wpos % windowSize
 			out := win[wpos]
 			win[wpos] = b
 			digest ^= uint64(tabout[out])
-			wpos = (wpos + 1) % windowSize
+			wpos++
 
 			// updateDigest
 			index := byte(digest >> polShift)
@@ -331,7 +344,7 @@ func (c *Chunker) Next(data []byte) (Chunk, error) {
 		}
 		c.digest = digest
 		c.window = win
-		c.wpos = wpos
+		c.wpos = wpos % windowSize
 
 		steps := c.bmax - c.bpos
 		if steps > 0 {

chunker/chunker_test.go

@@ -0,0 +1,347 @@
+package chunker
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func parseDigest(s string) []byte {
+	d, err := hex.DecodeString(s)
+	if err != nil {
+		panic(err)
+	}
+
+	return d
+}
+
+type chunk struct {
+	Length uint
+	CutFP  uint64
+	Digest []byte
+}
+
+// polynomial used for all the tests below
+const testPol = Pol(0x3DA3358B4DC173)
+
+// created for 32MB of random data out of math/rand's Uint32() seeded by
+// constant 23
+//
+// chunking configuration:
+// window size 64, avg chunksize 1<<20, min chunksize 1<<19, max chunksize 1<<23
+// polynom 0x3DA3358B4DC173
+var chunks1 = []chunk{
+	chunk{2163460, 0x000b98d4cdf00000, parseDigest("4b94cb2cf293855ea43bf766731c74969b91aa6bf3c078719aabdd19860d590d")},
+	chunk{643703, 0x000d4e8364d00000, parseDigest("5727a63c0964f365ab8ed2ccf604912f2ea7be29759a2b53ede4d6841e397407")},
+	chunk{1528956, 0x0015a25c2ef00000, parseDigest("a73759636a1e7a2758767791c69e81b69fb49236c6929e5d1b654e06e37674ba")},
+	chunk{1955808, 0x00102a8242e00000, parseDigest("c955fb059409b25f07e5ae09defbbc2aadf117c97a3724e06ad4abd2787e6824")},
+	chunk{2222372, 0x00045da878000000, parseDigest("6ba5e9f7e1b310722be3627716cf469be941f7f3e39a4c3bcefea492ec31ee56")},
+	chunk{2538687, 0x00198a8179900000, parseDigest("8687937412f654b5cfe4a82b08f28393a0c040f77c6f95e26742c2fc4254bfde")},
+	chunk{609606, 0x001d4e8d17100000, parseDigest("5da820742ff5feb3369112938d3095785487456f65a8efc4b96dac4be7ebb259")},
+	chunk{1205738, 0x000a7204dd600000, parseDigest("cc70d8fad5472beb031b1aca356bcab86c7368f40faa24fe5f8922c6c268c299")},
+	chunk{959742, 0x00183e71e1400000, parseDigest("4065bdd778f95676c92b38ac265d361f81bff17d76e5d9452cf985a2ea5a4e39")},
+	chunk{4036109, 0x001fec043c700000, parseDigest("b9cf166e75200eb4993fc9b6e22300a6790c75e6b0fc8f3f29b68a752d42f275")},
+	chunk{1525894, 0x000b1574b1500000, parseDigest("2f238180e4ca1f7520a05f3d6059233926341090f9236ce677690c1823eccab3")},
+	chunk{1352720, 0x00018965f2e00000, parseDigest("afd12f13286a3901430de816e62b85cc62468c059295ce5888b76b3af9028d84")},
+	chunk{811884, 0x00155628aa100000, parseDigest("42d0cdb1ee7c48e552705d18e061abb70ae7957027db8ae8db37ec756472a70a")},
+	chunk{1282314, 0x001909a0a1400000, parseDigest("819721c2457426eb4f4c7565050c44c32076a56fa9b4515a1c7796441730eb58")},
+	chunk{1318021, 0x001cceb980000000, parseDigest("842eb53543db55bacac5e25cb91e43cc2e310fe5f9acc1aee86bdf5e91389374")},
+	chunk{948640, 0x0011f7a470a00000, parseDigest("b8e36bf7019bb96ac3fb7867659d2167d9d3b3148c09fe0de45850b8fe577185")},
+	chunk{645464, 0x00030ce2d9400000, parseDigest("5584bd27982191c3329f01ed846bfd266e96548dfa87018f745c33cfc240211d")},
+	chunk{533758, 0x0004435c53c00000, parseDigest("4da778a25b72a9a0d53529eccfe2e5865a789116cb1800f470d8df685a8ab05d")},
+	chunk{1128303, 0x0000c48517800000, parseDigest("08c6b0b38095b348d80300f0be4c5184d2744a17147c2cba5cc4315abf4c048f")},
+	chunk{800374, 0x000968473f900000, parseDigest("820284d2c8fd243429674c996d8eb8d3450cbc32421f43113e980f516282c7bf")},
+	chunk{2453512, 0x001e197c92600000, parseDigest("5fa870ed107c67704258e5e50abe67509fb73562caf77caa843b5f243425d853")},
+	chunk{2651975, 0x000ae6c868000000, parseDigest("181347d2bbec32bef77ad5e9001e6af80f6abcf3576549384d334ee00c1988d8")},
+	chunk{237392, 0x0000000000000001, parseDigest("fcd567f5d866357a8e299fd5b2359bb2c8157c30395229c4e9b0a353944a7978")},
+}
+
+// test if nullbytes are correctly split, even if length is a multiple of MinSize.
+var chunks2 = []chunk{
+	chunk{MinSize, 0, parseDigest("07854d2fef297a06ba81685e660c332de36d5d18d546927d30daad6d7fda1541")},
+	chunk{MinSize, 0, parseDigest("07854d2fef297a06ba81685e660c332de36d5d18d546927d30daad6d7fda1541")},
+	chunk{MinSize, 0, parseDigest("07854d2fef297a06ba81685e660c332de36d5d18d546927d30daad6d7fda1541")},
+	chunk{MinSize, 0, parseDigest("07854d2fef297a06ba81685e660c332de36d5d18d546927d30daad6d7fda1541")},
+}
+
+// the same as chunks1, but avg chunksize is 1<<19
+var chunks3 = []chunk{
+	chunk{1491586, 0x00023e586ea80000, parseDigest("4c008237df602048039287427171cef568a6cb965d1b5ca28dc80504a24bb061")},
+	chunk{671874, 0x000b98d4cdf00000, parseDigest("fa8a42321b90c3d4ce9dd850562b2fd0c0fe4bdd26cf01a24f22046a224225d3")},
+	chunk{643703, 0x000d4e8364d00000, parseDigest("5727a63c0964f365ab8ed2ccf604912f2ea7be29759a2b53ede4d6841e397407")},
+	chunk{1284146, 0x0012b527e4780000, parseDigest("16d04cafecbeae9eaedd49da14c7ad7cdc2b1cc8569e5c16c32c9fb045aa899a")},
+	chunk{823366, 0x000d1d6752180000, parseDigest("48662c118514817825ad4761e8e2e5f28f9bd8281b07e95dcafc6d02e0aa45c3")},
+	chunk{810134, 0x0016071b6e180000, parseDigest("f629581aa05562f97f2c359890734c8574c5575da32f9289c5ba70bfd05f3f46")},
+	chunk{567118, 0x00102a8242e00000, parseDigest("d4f0797c56c60d01bac33bfd49957a4816b6c067fc155b026de8a214cab4d70a")},
+	chunk{821315, 0x001b3e42c8180000, parseDigest("8ebd0fd5db0293bd19140da936eb8b1bbd3cd6ffbec487385b956790014751ca")},
+	chunk{1401057, 0x00045da878000000, parseDigest("001360af59adf4871ef138cfa2bb49007e86edaf5ac2d6f0b3d3014510991848")},
+	chunk{2311122, 0x0005cbd885380000, parseDigest("8276d489b566086d9da95dc5c5fe6fc7d72646dd3308ced6b5b6ddb8595f0aa1")},
+	chunk{608723, 0x001cfcd86f280000, parseDigest("518db33ba6a79d4f3720946f3785c05b9611082586d47ea58390fc2f6de9449e")},
+	chunk{980456, 0x0013edb7a7f80000, parseDigest("0121b1690738395e15fecba1410cd0bf13fde02225160cad148829f77e7b6c99")},
+	chunk{1140278, 0x0001f9f017e80000, parseDigest("28ca7c74804b5075d4f5eeb11f0845d99f62e8ea3a42b9a05c7bd5f2fca619dd")},
+	chunk{2015542, 0x00097bf5d8180000, parseDigest("6fe8291f427d48650a5f0f944305d3a2dbc649bd401d2655fc0bdd42e890ca5a")},
+	chunk{904752, 0x000e1863eff80000, parseDigest("62af1f1eb3f588d18aff28473303cc4731fc3cafcc52ce818fee3c4c2820854d")},
+	chunk{713072, 0x001f3bb1b9b80000, parseDigest("4bda9dc2e3031d004d87a5cc93fe5207c4b0843186481b8f31597dc6ffa1496c")},
+	chunk{675937, 0x001fec043c700000, parseDigest("5299c8c5acec1b90bb020cd75718aab5e12abb9bf66291465fd10e6a823a8b4a")},
+	chunk{1525894, 0x000b1574b1500000, parseDigest("2f238180e4ca1f7520a05f3d6059233926341090f9236ce677690c1823eccab3")},
+	chunk{1352720, 0x00018965f2e00000, parseDigest("afd12f13286a3901430de816e62b85cc62468c059295ce5888b76b3af9028d84")},
+	chunk{811884, 0x00155628aa100000, parseDigest("42d0cdb1ee7c48e552705d18e061abb70ae7957027db8ae8db37ec756472a70a")},
+	chunk{1282314, 0x001909a0a1400000, parseDigest("819721c2457426eb4f4c7565050c44c32076a56fa9b4515a1c7796441730eb58")},
+	chunk{1093738, 0x0017f5d048880000, parseDigest("5dddfa7a241b68f65d267744bdb082ee865f3c2f0d8b946ea0ee47868a01bbff")},
+	chunk{962003, 0x000b921f7ef80000, parseDigest("0cb5c9ebba196b441c715c8d805f6e7143a81cd5b0d2c65c6aacf59ca9124af9")},
+	chunk{856384, 0x00030ce2d9400000, parseDigest("7734b206d46f3f387e8661e81edf5b1a91ea681867beb5831c18aaa86632d7fb")},
+	chunk{533758, 0x0004435c53c00000, parseDigest("4da778a25b72a9a0d53529eccfe2e5865a789116cb1800f470d8df685a8ab05d")},
+	chunk{1128303, 0x0000c48517800000, parseDigest("08c6b0b38095b348d80300f0be4c5184d2744a17147c2cba5cc4315abf4c048f")},
+	chunk{800374, 0x000968473f900000, parseDigest("820284d2c8fd243429674c996d8eb8d3450cbc32421f43113e980f516282c7bf")},
+	chunk{2453512, 0x001e197c92600000, parseDigest("5fa870ed107c67704258e5e50abe67509fb73562caf77caa843b5f243425d853")},
+	chunk{665901, 0x00118c842cb80000, parseDigest("deceec26163842fdef6560311c69bf8a9871a56e16d719e2c4b7e4d668ceb61f")},
+	chunk{1986074, 0x000ae6c868000000, parseDigest("64cd64bf3c3bc389eb20df8310f0427d1c36ab2eaaf09e346bfa7f0453fc1a18")},
+	chunk{237392, 0x0000000000000001, parseDigest("fcd567f5d866357a8e299fd5b2359bb2c8157c30395229c4e9b0a353944a7978")},
+}
+
+func testWithData(t *testing.T, chnker *Chunker, testChunks []chunk, checkDigest bool) []Chunk {
+	chunks := []Chunk{}
+
+	pos := uint(0)
+	for i, chunk := range testChunks {
+		c, err := chnker.Next(nil)
+
+		if err != nil {
+			t.Fatalf("Error returned with chunk %d: %v", i, err)
+		}
+
+		if c.Start != pos {
+			t.Fatalf("Start for chunk %d does not match: expected %d, got %d",
+				i, pos, c.Start)
+		}
+
+		if c.Length != chunk.Length {
+			t.Fatalf("Length for chunk %d does not match: expected %d, got %d",
+				i, chunk.Length, c.Length)
+		}
+
+		if c.Cut != chunk.CutFP {
+			t.Fatalf("Cut fingerprint for chunk %d/%d does not match: expected %016x, got %016x",
+				i, len(chunks)-1, chunk.CutFP, c.Cut)
+		}
+
+		if checkDigest {
+			digest := hashData(c.Data)
+			if !bytes.Equal(chunk.Digest, digest) {
+				t.Fatalf("Digest fingerprint for chunk %d/%d does not match: expected %02x, got %02x",
+					i, len(chunks)-1, chunk.Digest, digest)
+			}
+		}
+
+		pos += c.Length
+		chunks = append(chunks, c)
+	}
+
+	_, err := chnker.Next(nil)
+	if err != io.EOF {
+		t.Fatal("Wrong error returned after last chunk")
+	}
+
+	if len(chunks) != len(testChunks) {
+		t.Fatal("Amounts of test and resulting chunks do not match")
+	}
+
+	return chunks
+}
+
+func getRandom(seed int64, count int) []byte {
+	buf := make([]byte, count)
+
+	rnd := rand.New(rand.NewSource(seed))
+	for i := 0; i < count; i += 4 {
+		r := rnd.Uint32()
+		buf[i] = byte(r)
+		buf[i+1] = byte(r >> 8)
+		buf[i+2] = byte(r >> 16)
+		buf[i+3] = byte(r >> 24)
+	}
+
+	return buf
+}
+
+func hashData(d []byte) []byte {
+	h := sha256.New()
+	h.Write(d)
+	return h.Sum(nil)
+}
+
+func TestChunker(t *testing.T) {
+	// setup data source
+	buf := getRandom(23, 32*1024*1024)
+	ch := New(bytes.NewReader(buf), testPol)
+	testWithData(t, ch, chunks1, true)
+
+	// setup nullbyte data source
+	buf = bytes.Repeat([]byte{0}, len(chunks2)*MinSize)
+	ch = New(bytes.NewReader(buf), testPol)
+
+	testWithData(t, ch, chunks2, true)
+}
+
+func TestChunkerWithCustomAverageBits(t *testing.T) {
+	buf := getRandom(23, 32*1024*1024)
+	ch := New(bytes.NewReader(buf), testPol)
+
+	// sligthly decrease averageBits to get more chunks
+	ch.SetAverageBits(19)
+	testWithData(t, ch, chunks3, true)
+}
+
+func TestChunkerReset(t *testing.T) {
+	buf := getRandom(23, 32*1024*1024)
+	ch := New(bytes.NewReader(buf), testPol)
+	testWithData(t, ch, chunks1, true)
+
+	ch.Reset(bytes.NewReader(buf), testPol)
+	testWithData(t, ch, chunks1, true)
+}
+
+func TestChunkerWithRandomPolynomial(t *testing.T) {
+	// setup data source
+	buf := getRandom(23, 32*1024*1024)
+
+	// generate a new random polynomial
+	start := time.Now()
+	p, err := RandomPolynomial()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("generating random polynomial took %v", time.Since(start))
+
+	start = time.Now()
+	ch := New(bytes.NewReader(buf), p)
+	t.Logf("creating chunker took %v", time.Since(start))
+
+	// make sure that first chunk is different
+	c, err := ch.Next(nil)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Cut == chunks1[0].CutFP {
+		t.Fatal("Cut point is the same")
+	}
+
+	if c.Length == chunks1[0].Length {
+		t.Fatal("Length is the same")
+	}
+
+	if bytes.Equal(hashData(c.Data), chunks1[0].Digest) {
+		t.Fatal("Digest is the same")
+	}
+}
+
+func TestChunkerWithoutHash(t *testing.T) {
+	// setup data source
+	buf := getRandom(23, 32*1024*1024)
+
+	ch := New(bytes.NewReader(buf), testPol)
+	chunks := testWithData(t, ch, chunks1, false)
+
+	// test reader
+	for i, c := range chunks {
+		if uint(len(c.Data)) != chunks1[i].Length {
+			t.Fatalf("reader returned wrong number of bytes: expected %d, got %d",
+				chunks1[i].Length, len(c.Data))
+		}
+
+		if !bytes.Equal(buf[c.Start:c.Start+c.Length], c.Data) {
+			t.Fatalf("invalid data for chunk returned: expected %02x, got %02x",
+				buf[c.Start:c.Start+c.Length], c.Data)
+		}
+	}
+
+	// setup nullbyte data source
+	buf = bytes.Repeat([]byte{0}, len(chunks2)*MinSize)
+	ch = New(bytes.NewReader(buf), testPol)
+
+	testWithData(t, ch, chunks2, false)
+}
+
+func benchmarkChunker(b *testing.B, checkDigest bool) {
+	size := 32 * 1024 * 1024
+	rd := bytes.NewReader(getRandom(23, size))
+	ch := New(rd, testPol)
+	buf := make([]byte, MaxSize)
+
+	b.ResetTimer()
+	b.SetBytes(int64(size))
+
+	var chunks int
+	for i := 0; i < b.N; i++ {
+		chunks = 0
+
+		_, err := rd.Seek(0, 0)
+		if err != nil {
+			b.Fatalf("Seek() return error %v", err)
+		}
+
+		ch.Reset(rd, testPol)
+
+		cur := 0
+		for {
+			chunk, err := ch.Next(buf)
+
+			if err == io.EOF {
+				break
+			}
+
+			if err != nil {
+				b.Fatalf("Unexpected error occurred: %v", err)
+			}
+
+			if chunk.Length != chunks1[cur].Length {
+				b.Errorf("wrong chunk length, want %d, got %d",
+					chunks1[cur].Length, chunk.Length)
+			}
+
+			if chunk.Cut != chunks1[cur].CutFP {
+				b.Errorf("wrong cut fingerprint, want 0x%x, got 0x%x",
+					chunks1[cur].CutFP, chunk.Cut)
+			}
+
+			if checkDigest {
+				h := hashData(chunk.Data)
+				if !bytes.Equal(h, chunks1[cur].Digest) {
+					b.Errorf("wrong digest, want %x, got %x",
+						chunks1[cur].Digest, h)
+				}
+			}
+
+			chunks++
+			cur++
+		}
+	}
+
+	b.Logf("%d chunks, average chunk size: %d bytes", chunks, size/chunks)
+}
+
+func BenchmarkChunkerWithSHA256(b *testing.B) {
+	benchmarkChunker(b, true)
+}
+
+func BenchmarkChunker(b *testing.B) {
+	benchmarkChunker(b, false)
+}
+
+func BenchmarkNewChunker(b *testing.B) {
+	p, err := RandomPolynomial()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		New(bytes.NewBuffer(nil), p)
+	}
+}

chunker/example_test.go

@@ -0,0 +1,39 @@
+package chunker
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"fmt"
+	"io"
+)
+
+func ExampleChunker() {
+	// generate 32MiB of deterministic pseudo-random data
+	data := getRandom(23, 32*1024*1024)
+
+	// create a chunker
+	chunker := New(bytes.NewReader(data), Pol(0x3DA3358B4DC173))
+
+	// reuse this buffer
+	buf := make([]byte, 8*1024*1024)
+
+	for i := 0; i < 5; i++ {
+		chunk, err := chunker.Next(buf)
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			panic(err)
+		}
+
+		fmt.Printf("%d %02x\n", chunk.Length, sha256.Sum256(chunk.Data))
+	}
+
+	// Output:
+	// 2163460 4b94cb2cf293855ea43bf766731c74969b91aa6bf3c078719aabdd19860d590d
+	// 643703 5727a63c0964f365ab8ed2ccf604912f2ea7be29759a2b53ede4d6841e397407
+	// 1528956 a73759636a1e7a2758767791c69e81b69fb49236c6929e5d1b654e06e37674ba
+	// 1955808 c955fb059409b25f07e5ae09defbbc2aadf117c97a3724e06ad4abd2787e6824
+	// 2222372 6ba5e9f7e1b310722be3627716cf469be941f7f3e39a4c3bcefea492ec31ee56
+}

chunker/go.mod

@@ -0,0 +1,3 @@
+module github.com/restic/chunker
+
+go 1.14

chunker/polynomials.go

@@ -94,7 +94,6 @@ func (x Pol) Deg() int {
 	}
 
 	if uint64(x)&0x2 > 0 {
-		x >>= 1
 		r |= 1
 	}
 

chunker/polynomials_test.go

@@ -0,0 +1,424 @@
+package chunker
+
+import (
+	"strconv"
+	"testing"
+)
+
+var polAddTests = []struct {
+	x, y Pol
+	sum  Pol
+}{
+	{23, 16, 23 ^ 16},
+	{0x9a7e30d1e855e0a0, 0x670102a1f4bcd414, 0xfd7f32701ce934b4},
+	{0x9a7e30d1e855e0a0, 0x9a7e30d1e855e0a0, 0},
+}
+
+func TestPolAdd(t *testing.T) {
+	for i, test := range polAddTests {
+		if test.sum != test.x.Add(test.y) {
+			t.Errorf("test %d failed: sum != x+y", i)
+		}
+
+		if test.sum != test.y.Add(test.x) {
+			t.Errorf("test %d failed: sum != y+x", i)
+		}
+	}
+}
+
+func parseBin(s string) Pol {
+	i, err := strconv.ParseUint(s, 2, 64)
+	if err != nil {
+		panic(err)
+	}
+
+	return Pol(i)
+}
+
+var polMulTests = []struct {
+	x, y Pol
+	res  Pol
+}{
+	{1, 2, 2},
+	{
+		parseBin("1101"),
+		parseBin("10"),
+		parseBin("11010"),
+	},
+	{
+		parseBin("1101"),
+		parseBin("11"),
+		parseBin("10111"),
+	},
+	{
+		0x40000000,
+		0x40000000,
+		0x1000000000000000,
+	},
+	{
+		parseBin("1010"),
+		parseBin("100100"),
+		parseBin("101101000"),
+	},
+	{
+		parseBin("100"),
+		parseBin("11"),
+		parseBin("1100"),
+	},
+	{
+		parseBin("11"),
+		parseBin("110101"),
+		parseBin("1011111"),
+	},
+	{
+		parseBin("10011"),
+		parseBin("110101"),
+		parseBin("1100001111"),
+	},
+}
+
+func TestPolMul(t *testing.T) {
+	for i, test := range polMulTests {
+		m := test.x.Mul(test.y)
+		if test.res != m {
+			t.Errorf("TestPolMul failed for test %d: %v * %v: want %v, got %v",
+				i, test.x, test.y, test.res, m)
+		}
+		m = test.y.Mul(test.x)
+		if test.res != test.y.Mul(test.x) {
+			t.Errorf("TestPolMul failed for %d: %v * %v: want %v, got %v",
+				i, test.x, test.y, test.res, m)
+		}
+	}
+}
+
+func TestPolMulOverflow(t *testing.T) {
+	defer func() {
+		// try to recover overflow error
+		err := recover()
+
+		if e, ok := err.(string); ok && e == "multiplication would overflow uint64" {
+			return
+		}
+
+		t.Logf("invalid error raised: %v", err)
+		// re-raise error if not overflow
+		panic(err)
+	}()
+
+	x := Pol(1 << 63)
+	x.Mul(2)
+	t.Fatal("overflow test did not panic")
+}
+
+var polDivTests = []struct {
+	x, y Pol
+	res  Pol
+}{
+	{10, 50, 0},
+	{0, 1, 0},
+	{
+		parseBin("101101000"), // 0x168
+		parseBin("1010"),      // 0xa
+		parseBin("100100"),    // 0x24
+	},
+	{2, 2, 1},
+	{
+		0x8000000000000000,
+		0x8000000000000000,
+		1,
+	},
+	{
+		parseBin("1100"),
+		parseBin("100"),
+		parseBin("11"),
+	},
+	{
+		parseBin("1100001111"),
+		parseBin("10011"),
+		parseBin("110101"),
+	},
+}
+
+func TestPolDiv(t *testing.T) {
+	for i, test := range polDivTests {
+		m := test.x.Div(test.y)
+		if test.res != m {
+			t.Errorf("TestPolDiv failed for test %d: %v * %v: want %v, got %v",
+				i, test.x, test.y, test.res, m)
+		}
+	}
+}
+
+func TestPolDeg(t *testing.T) {
+	var x Pol
+	if x.Deg() != -1 {
+		t.Errorf("deg(0) is not -1: %v", x.Deg())
+	}
+
+	x = 1
+	if x.Deg() != 0 {
+		t.Errorf("deg(1) is not 0: %v", x.Deg())
+	}
+
+	for i := 0; i < 64; i++ {
+		x = 1 << uint(i)
+		if x.Deg() != i {
+			t.Errorf("deg(1<<%d) is not %d: %v", i, i, x.Deg())
+		}
+	}
+}
+
+var polModTests = []struct {
+	x, y Pol
+	res  Pol
+}{
+	{10, 50, 10},
+	{0, 1, 0},
+	{
+		parseBin("101101001"),
+		parseBin("1010"),
+		parseBin("1"),
+	},
+	{2, 2, 0},
+	{
+		0x8000000000000000,
+		0x8000000000000000,
+		0,
+	},
+	{
+		parseBin("1100"),
+		parseBin("100"),
+		parseBin("0"),
+	},
+	{
+		parseBin("1100001111"),
+		parseBin("10011"),
+		parseBin("0"),
+	},
+}
+
+func TestPolModt(t *testing.T) {
+	for i, test := range polModTests {
+		res := test.x.Mod(test.y)
+		if test.res != res {
+			t.Errorf("test %d failed: want %v, got %v", i, test.res, res)
+		}
+	}
+}
+
+func BenchmarkPolDivMod(t *testing.B) {
+	f := Pol(0x2482734cacca49)
+	g := Pol(0x3af4b284899)
+
+	for i := 0; i < t.N; i++ {
+		g.DivMod(f)
+	}
+}
+
+func BenchmarkPolDiv(t *testing.B) {
+	f := Pol(0x2482734cacca49)
+	g := Pol(0x3af4b284899)
+
+	for i := 0; i < t.N; i++ {
+		g.Div(f)
+	}
+}
+
+func BenchmarkPolMod(t *testing.B) {
+	f := Pol(0x2482734cacca49)
+	g := Pol(0x3af4b284899)
+
+	for i := 0; i < t.N; i++ {
+		g.Mod(f)
+	}
+}
+
+func BenchmarkPolDeg(t *testing.B) {
+	f := Pol(0x3af4b284899)
+	d := f.Deg()
+	if d != 41 {
+		t.Fatalf("BenchmalPolDeg: Wrong degree %d returned, expected %d",
+			d, 41)
+	}
+
+	for i := 0; i < t.N; i++ {
+		f.Deg()
+	}
+}
+
+func TestRandomPolynomial(t *testing.T) {
+	_, err := RandomPolynomial()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func BenchmarkRandomPolynomial(t *testing.B) {
+	for i := 0; i < t.N; i++ {
+		_, err := RandomPolynomial()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestExpandPolynomial(t *testing.T) {
+	pol := Pol(0x3DA3358B4DC173)
+	s := pol.Expand()
+	if s != "x^53+x^52+x^51+x^50+x^48+x^47+x^45+x^41+x^40+x^37+x^36+x^34+x^32+x^31+x^27+x^25+x^24+x^22+x^19+x^18+x^16+x^15+x^14+x^8+x^6+x^5+x^4+x+1" {
+		t.Fatal("wrong result")
+	}
+}
+
+var polIrredTests = []struct {
+	f     Pol
+	irred bool
+}{
+	{0x38f1e565e288df, false},
+	{0x3DA3358B4DC173, true},
+	{0x30a8295b9d5c91, false},
+	{0x255f4350b962cb, false},
+	{0x267f776110a235, false},
+	{0x2f4dae10d41227, false},
+	{0x2482734cacca49, true},
+	{0x312daf4b284899, false},
+	{0x29dfb6553d01d1, false},
+	{0x3548245eb26257, false},
+	{0x3199e7ef4211b3, false},
+	{0x362f39017dae8b, false},
+	{0x200d57aa6fdacb, false},
+	{0x35e0a4efa1d275, false},
+	{0x2ced55b026577f, false},
+	{0x260b012010893d, false},
+	{0x2df29cbcd59e9d, false},
+	{0x3f2ac7488bd429, false},
+	{0x3e5cb1711669fb, false},
+	{0x226d8de57a9959, false},
+	{0x3c8de80aaf5835, false},
+	{0x2026a59efb219b, false},
+	{0x39dfa4d13fb231, false},
+	{0x3143d0464b3299, false},
+}
+
+func TestPolIrreducible(t *testing.T) {
+	for _, test := range polIrredTests {
+		if test.f.Irreducible() != test.irred {
+			t.Errorf("Irreducibility test for Polynomial %v failed: got %v, wanted %v",
+				test.f, test.f.Irreducible(), test.irred)
+		}
+	}
+}
+
+func BenchmarkPolIrreducible(b *testing.B) {
+	// find first irreducible polynomial
+	var pol Pol
+	for _, test := range polIrredTests {
+		if test.irred {
+			pol = test.f
+			break
+		}
+	}
+
+	for i := 0; i < b.N; i++ {
+		if !pol.Irreducible() {
+			b.Errorf("Irreducibility test for Polynomial %v failed", pol)
+		}
+	}
+}
+
+var polGCDTests = []struct {
+	f1  Pol
+	f2  Pol
+	gcd Pol
+}{
+	{10, 50, 2},
+	{0, 1, 1},
+	{
+		parseBin("101101001"),
+		parseBin("1010"),
+		parseBin("1"),
+	},
+	{2, 2, 2},
+	{
+		parseBin("1010"),
+		parseBin("11"),
+		parseBin("11"),
+	},
+	{
+		0x8000000000000000,
+		0x8000000000000000,
+		0x8000000000000000,
+	},
+	{
+		parseBin("1100"),
+		parseBin("101"),
+		parseBin("11"),
+	},
+	{
+		parseBin("1100001111"),
+		parseBin("10011"),
+		parseBin("10011"),
+	},
+	{
+		0x3DA3358B4DC173,
+		0x3DA3358B4DC173,
+		0x3DA3358B4DC173,
+	},
+	{
+		0x3DA3358B4DC173,
+		0x230d2259defd,
+		1,
+	},
+	{
+		0x230d2259defd,
+		0x51b492b3eff2,
+		parseBin("10011"),
+	},
+}
+
+func TestPolGCD(t *testing.T) {
+	for i, test := range polGCDTests {
+		gcd := test.f1.GCD(test.f2)
+		if test.gcd != gcd {
+			t.Errorf("GCD test %d (%+v) failed: got %v, wanted %v",
+				i, test, gcd, test.gcd)
+		}
+
+		gcd = test.f2.GCD(test.f1)
+		if test.gcd != gcd {
+			t.Errorf("GCD test %d (%+v) failed: got %v, wanted %v",
+				i, test, gcd, test.gcd)
+		}
+	}
+}
+
+var polMulModTests = []struct {
+	f1  Pol
+	f2  Pol
+	g   Pol
+	mod Pol
+}{
+	{
+		0x1230,
+		0x230,
+		0x55,
+		0x22,
+	},
+	{
+		0x0eae8c07dbbb3026,
+		0xd5d6db9de04771de,
+		0xdd2bda3b77c9,
+		0x425ae8595b7a,
+	},
+}
+
+func TestPolMulMod(t *testing.T) {
+	for i, test := range polMulModTests {
+		mod := test.f1.MulMod(test.f2, test.g)
+		if mod != test.mod {
+			t.Errorf("MulMod test %d (%+v) failed: got %v, wanted %v",
+				i, test, mod, test.mod)
+		}
+	}
+}

cmd/restic/acl.go

@@ -0,0 +1,131 @@
+package main
+
+// Adapted from https://github.com/maxymania/go-system/blob/master/posix_acl/posix_acl.go
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+)
+
+const (
+	aclUserOwner  = 0x0001
+	aclUser       = 0x0002
+	aclGroupOwner = 0x0004
+	aclGroup      = 0x0008
+	aclMask       = 0x0010
+	aclOthers     = 0x0020
+)
+
+type aclSID uint64
+
+type aclElem struct {
+	Tag  uint16
+	Perm uint16
+	ID   uint32
+}
+
+type acl struct {
+	Version uint32
+	List    []aclElement
+}
+
+type aclElement struct {
+	aclSID
+	Perm uint16
+}
+
+func (a *aclSID) setUID(uid uint32) {
+	*a = aclSID(uid) | (aclUser << 32)
+}
+func (a *aclSID) setGID(gid uint32) {
+	*a = aclSID(gid) | (aclGroup << 32)
+}
+
+func (a *aclSID) setType(tp int) {
+	*a = aclSID(tp) << 32
+}
+
+func (a aclSID) getType() int {
+	return int(a >> 32)
+}
+func (a aclSID) getID() uint32 {
+	return uint32(a & 0xffffffff)
+}
+func (a aclSID) String() string {
+	switch a >> 32 {
+	case aclUserOwner:
+		return "user::"
+	case aclUser:
+		return fmt.Sprintf("user:%v:", a.getID())
+	case aclGroupOwner:
+		return "group::"
+	case aclGroup:
+		return fmt.Sprintf("group:%v:", a.getID())
+	case aclMask:
+		return "mask::"
+	case aclOthers:
+		return "other::"
+	}
+	return "?:"
+}
+
+func (a aclElement) String() string {
+	str := ""
+	if (a.Perm & 4) != 0 {
+		str += "r"
+	} else {
+		str += "-"
+	}
+	if (a.Perm & 2) != 0 {
+		str += "w"
+	} else {
+		str += "-"
+	}
+	if (a.Perm & 1) != 0 {
+		str += "x"
+	} else {
+		str += "-"
+	}
+	return fmt.Sprintf("%v%v", a.aclSID, str)
+}
+
+func (a *acl) decode(xattr []byte) {
+	var elem aclElement
+	ae := new(aclElem)
+	nr := bytes.NewReader(xattr)
+	e := binary.Read(nr, binary.LittleEndian, &a.Version)
+	if e != nil {
+		a.Version = 0
+		return
+	}
+	if len(a.List) > 0 {
+		a.List = a.List[:0]
+	}
+	for binary.Read(nr, binary.LittleEndian, ae) == nil {
+		elem.aclSID = (aclSID(ae.Tag) << 32) | aclSID(ae.ID)
+		elem.Perm = ae.Perm
+		a.List = append(a.List, elem)
+	}
+}
+
+func (a *acl) encode() []byte {
+	buf := new(bytes.Buffer)
+	ae := new(aclElem)
+	binary.Write(buf, binary.LittleEndian, &a.Version)
+	for _, elem := range a.List {
+		ae.Tag = uint16(elem.getType())
+		ae.Perm = elem.Perm
+		ae.ID = elem.getID()
+		binary.Write(buf, binary.LittleEndian, ae)
+	}
+	return buf.Bytes()
+}
+
+func (a *acl) String() string {
+	var finalacl string
+	for _, acl := range a.List {
+		finalacl += acl.String() + "\n"
+	}
+	return finalacl
+}

cmd/restic/acl_test.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_acl_decode(t *testing.T) {
+	type args struct {
+		xattr []byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "decode string",
+			args: args{
+				xattr: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255},
+			},
+			want: "user::rw-\nuser:0:rwx\nuser:65534:rwx\ngroup::rwx\nmask::rwx\nother::r--\n",
+		},
+		{
+			name: "decode fail",
+			args: args{
+				xattr: []byte("abctest"),
+			},
+			want: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := &acl{}
+			a.decode(tt.args.xattr)
+			if tt.want != a.String() {
+				t.Errorf("acl.decode() = %v, want: %v", a.String(), tt.want)
+			}
+		})
+	}
+}
+
+func Test_acl_encode(t *testing.T) {
+	tests := []struct {
+		name string
+		want []byte
+		args []aclElement
+	}{
+		{
+			name: "encode values",
+			want: []byte{2, 0, 0, 0, 1, 0, 6, 0, 255, 255, 255, 255, 2, 0, 7, 0, 0, 0, 0, 0, 2, 0, 7, 0, 254, 255, 0, 0, 4, 0, 7, 0, 255, 255, 255, 255, 16, 0, 7, 0, 255, 255, 255, 255, 32, 0, 4, 0, 255, 255, 255, 255},
+			args: []aclElement{
+				{
+					aclSID: 8589934591,
+					Perm:   6,
+				},
+				{
+					aclSID: 8589934592,
+					Perm:   7,
+				},
+				{
+					aclSID: 8590000126,
+					Perm:   7,
+				},
+				{
+					aclSID: 21474836479,
+					Perm:   7,
+				},
+				{
+					aclSID: 73014444031,
+					Perm:   7,
+				},
+				{
+					aclSID: 141733920767,
+					Perm:   4,
+				},
+			},
+		},
+		{
+			name: "encode fail",
+			want: []byte{2, 0, 0, 0},
+			args: []aclElement{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			a := &acl{
+				Version: 2,
+				List:    tt.args,
+			}
+			if got := a.encode(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("acl.encode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/restic/cleanup.go

@@ -20,7 +20,7 @@ var cleanupHandlers struct {
 var stderr = os.Stderr
 
 func init() {
-	cleanupHandlers.ch = make(chan os.Signal)
+	cleanupHandlers.ch = make(chan os.Signal, 1)
 	go CleanupHandler(cleanupHandlers.ch)
 	signal.Notify(cleanupHandlers.ch, syscall.SIGINT)
 }

cmd/restic/cmd_backup.go

@@ -4,8 +4,12 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -21,6 +25,7 @@ import (
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
 	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/json"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
 
@@ -30,21 +35,33 @@ var cmdBackup = &cobra.Command{
 	Long: `
 The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+
+Note that some issues such as unreadable or deleted files during backup
+currently doesn't result in a non-zero error exit status.
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		if backupOptions.Hostname == "" {
+		if backupOptions.Host == "" {
 			hostname, err := os.Hostname()
 			if err != nil {
 				debug.Log("os.Hostname() returned err: %v", err)
 				return
 			}
-			backupOptions.Hostname = hostname
+			backupOptions.Host = hostname
 		}
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
-			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
+		if backupOptions.Stdin {
+			for _, filename := range backupOptions.FilesFrom {
+				if filename == "-" {
+					return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
+				}
+			}
 		}
 
 		var t tomb.Tomb
@@ -62,20 +79,22 @@ given as the arguments.
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	Parent           string
-	Force            bool
-	Excludes         []string
-	ExcludeFiles     []string
-	ExcludeOtherFS   bool
-	ExcludeIfPresent []string
-	ExcludeCaches    bool
-	Stdin            bool
-	StdinFilename    string
-	Tags             []string
-	Hostname         string
-	FilesFrom        string
-	TimeStamp        string
-	WithAtime        bool
+	Parent              string
+	Force               bool
+	Excludes            []string
+	InsensitiveExcludes []string
+	ExcludeFiles        []string
+	ExcludeOtherFS      bool
+	ExcludeIfPresent    []string
+	ExcludeCaches       bool
+	Stdin               bool
+	StdinFilename       string
+	Tags                []string
+	Host                string
+	FilesFrom           []string
+	TimeStamp           string
+	WithAtime           bool
+	IgnoreInode         bool
 }
 
 var backupOptions BackupOptions
@@ -84,20 +103,26 @@ func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
 	f := cmdBackup.Flags()
-	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
+	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repo that has the same target files/directories)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
 	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
-	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
-	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file`)
+	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
+	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See http://bford.info/cachedir/spec.html for the Cache Directory Tagging Standard`)
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
-	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
+	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
 	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
-	f.StringVar(&backupOptions.Hostname, "hostname", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
-	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
-	f.StringVar(&backupOptions.TimeStamp, "time", "", "time of the backup (ex. '2012-11-01 22:08:41') (default: now)")
+
+	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
+	f.MarkDeprecated("hostname", "use --host")
+
+	f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args/can be specified multiple times)")
+	f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
+	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -169,12 +194,16 @@ func readLinesFromFile(filename string) ([]string, error) {
 
 // Check returns an error when an invalid combination of options was set.
 func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
-	if opts.FilesFrom == "-" && gopts.password == "" {
-		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
+	if gopts.password == "" {
+		for _, filename := range opts.FilesFrom {
+			if filename == "-" {
+				return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
+			}
+		}
 	}
 
 	if opts.Stdin {
-		if opts.FilesFrom != "" {
+		if len(opts.FilesFrom) > 0 {
 			return errors.Fatal("--stdin and --files-from cannot be used together")
 		}
 
@@ -186,18 +215,9 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 	return nil
 }
 
-// collectRejectFuncs returns a list of all functions which may reject data
-// from being saved in a snapshot
-func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
-	// allowed devices
-	if opts.ExcludeOtherFS && !opts.Stdin {
-		f, err := rejectByDevice(targets)
-		if err != nil {
-			return nil, err
-		}
-		fs = append(fs, f)
-	}
-
+// collectRejectByNameFuncs returns a list of all functions which may reject data
+// from being saved in a snapshot based on path only
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectByNameFunc, err error) {
 	// exclude restic cache
 	if repo.Cache != nil {
 		f, err := rejectResticCache(repo)
@@ -217,6 +237,10 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets
 		opts.Excludes = append(opts.Excludes, excludes...)
 	}
 
+	if len(opts.InsensitiveExcludes) > 0 {
+		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
+	}
+
 	if len(opts.Excludes) > 0 {
 		fs = append(fs, rejectByPattern(opts.Excludes))
 	}
@@ -237,6 +261,21 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets
 	return fs, nil
 }
 
+// collectRejectFuncs returns a list of all functions which may reject data
+// from being saved in a snapshot based on path and file info
+func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets []string) (fs []RejectFunc, err error) {
+	// allowed devices
+	if opts.ExcludeOtherFS && !opts.Stdin {
+		f, err := rejectByDevice(targets)
+		if err != nil {
+			return nil, err
+		}
+		fs = append(fs, f)
+	}
+
+	return fs, nil
+}
+
 // readExcludePatternsFromFiles reads all exclude files and returns the list of
 // exclude patterns. For each line, leading and trailing white space is removed
 // and comment lines are ignored. For each remaining pattern, environment
@@ -290,15 +329,31 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 		return nil, nil
 	}
 
-	fromfile, err := readLinesFromFile(opts.FilesFrom)
-	if err != nil {
-		return nil, err
+	var lines []string
+	for _, file := range opts.FilesFrom {
+		fromfile, err := readLinesFromFile(file)
+		if err != nil {
+			return nil, err
+		}
+
+		// expand wildcards
+		for _, line := range fromfile {
+			var expanded []string
+			expanded, err := filepath.Glob(line)
+			if err != nil {
+				return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
+			}
+			if len(expanded) == 0 {
+				Warnf("pattern %q does not match any files, skipping\n", line)
+			}
+			lines = append(lines, expanded...)
+		}
 	}
 
 	// merge files from files-from into normal args so we can reuse the normal
 	// args checks and have the ability to use both files-from and args at the
 	// same time
-	args = append(args, fromfile...)
+	args = append(args, lines...)
 	if len(args) == 0 && !opts.Stdin {
 		return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
 	}
@@ -327,7 +382,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Hostname)
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host})
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -351,7 +406,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 
 	timeStamp := time.Now()
 	if opts.TimeStamp != "" {
-		timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
+		timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
 		if err != nil {
 			return errors.Fatalf("error in time option: %v\n", err)
 		}
@@ -359,7 +414,43 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 
 	var t tomb.Tomb
 
-	p := ui.NewBackup(term, gopts.verbosity)
+	if gopts.verbosity >= 2 && !gopts.JSON {
+		term.Print("open repository\n")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	type ArchiveProgressReporter interface {
+		CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration)
+		StartFile(filename string)
+		CompleteBlob(filename string, bytes uint64)
+		ScannerError(item string, fi os.FileInfo, err error) error
+		ReportTotal(item string, s archiver.ScanStats)
+		SetMinUpdatePause(d time.Duration)
+		Run(ctx context.Context) error
+		Error(item string, fi os.FileInfo, err error) error
+		Finish(snapshotID restic.ID)
+
+		// ui.StdioWrapper
+		Stdout() io.WriteCloser
+		Stderr() io.WriteCloser
+
+		// ui.Message
+		E(msg string, args ...interface{})
+		P(msg string, args ...interface{})
+		V(msg string, args ...interface{})
+		VV(msg string, args ...interface{})
+	}
+
+	var p ArchiveProgressReporter
+	if gopts.JSON {
+		p = json.NewBackup(term, gopts.verbosity)
+	} else {
+		p = ui.NewBackup(term, gopts.verbosity)
+	}
 
 	// use the terminal for stdout/stderr
 	prevStdout, prevStderr := gopts.stdout, gopts.stderr
@@ -374,32 +465,36 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 			if fps > 60 {
 				fps = 60
 			}
-			p.MinUpdatePause = time.Second / time.Duration(fps)
+			p.SetMinUpdatePause(time.Second / time.Duration(fps))
 		}
 	}
 
 	t.Go(func() error { return p.Run(t.Context(gopts.ctx)) })
 
-	p.V("open repository")
-	repo, err := OpenRepository(gopts)
-	if err != nil {
-		return err
+	if !gopts.JSON {
+		p.V("lock repository")
 	}
-
-	p.V("lock repository")
 	lock, err := lockRepo(repo)
 	defer unlockRepo(lock)
 	if err != nil {
 		return err
 	}
 
-	// rejectFuncs collect functions that can reject items from the backup
+	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
+	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, targets)
+	if err != nil {
+		return err
+	}
+
+	// rejectFuncs collect functions that can reject items from the backup based on path and file info
 	rejectFuncs, err := collectRejectFuncs(opts, repo, targets)
 	if err != nil {
 		return err
 	}
 
-	p.V("load index files")
+	if !gopts.JSON {
+		p.V("load index files")
+	}
 	err = repo.LoadIndex(gopts.ctx)
 	if err != nil {
 		return err
@@ -410,10 +505,19 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return err
 	}
 
-	if parentSnapshotID != nil {
+	if !gopts.JSON && parentSnapshotID != nil {
 		p.V("using parent snapshot %v\n", parentSnapshotID.Str())
 	}
 
+	selectByNameFilter := func(item string) bool {
+		for _, reject := range rejectByNameFuncs {
+			if reject(item) {
+				return false
+			}
+		}
+		return true
+	}
+
 	selectFilter := func(item string, fi os.FileInfo) bool {
 		for _, reject := range rejectFuncs {
 			if reject(item, fi) {
@@ -425,31 +529,39 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 
 	var targetFS fs.FS = fs.Local{}
 	if opts.Stdin {
-		p.V("read data from stdin")
+		if !gopts.JSON {
+			p.V("read data from stdin")
+		}
+		filename := path.Join("/", opts.StdinFilename)
 		targetFS = &fs.Reader{
 			ModTime:    timeStamp,
-			Name:       opts.StdinFilename,
+			Name:       filename,
 			Mode:       0644,
 			ReadCloser: os.Stdin,
 		}
-		targets = []string{opts.StdinFilename}
+		targets = []string{filename}
 	}
 
 	sc := archiver.NewScanner(targetFS)
+	sc.SelectByName = selectByNameFilter
 	sc.Select = selectFilter
 	sc.Error = p.ScannerError
 	sc.Result = p.ReportTotal
 
-	p.V("start scan on %v", targets)
+	if !gopts.JSON {
+		p.V("start scan on %v", targets)
+	}
 	t.Go(func() error { return sc.Scan(t.Context(gopts.ctx), targets) })
 
 	arch := archiver.New(repo, targetFS, archiver.Options{})
+	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
 	arch.Error = p.Error
-	arch.CompleteItem = p.CompleteItemFn
+	arch.CompleteItem = p.CompleteItem
 	arch.StartFile = p.StartFile
 	arch.CompleteBlob = p.CompleteBlob
+	arch.IgnoreInode = opts.IgnoreInode
 
 	if parentSnapshotID == nil {
 		parentSnapshotID = &restic.ID{}
@@ -459,17 +571,21 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		Excludes:       opts.Excludes,
 		Tags:           opts.Tags,
 		Time:           timeStamp,
-		Hostname:       opts.Hostname,
+		Hostname:       opts.Host,
 		ParentSnapshot: *parentSnapshotID,
 	}
 
 	uploader := archiver.IndexUploader{
 		Repository: repo,
 		Start: func() {
-			p.VV("uploading intermediate index")
+			if !gopts.JSON {
+				p.VV("uploading intermediate index")
+			}
 		},
 		Complete: func(id restic.ID) {
-			p.V("uploaded intermediate index %v", id.Str())
+			if !gopts.JSON {
+				p.V("uploaded intermediate index %v", id.Str())
+			}
 		},
 	}
 
@@ -477,23 +593,26 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		return uploader.Upload(gopts.ctx, t.Context(gopts.ctx), 30*time.Second)
 	})
 
-	p.V("start backup on %v", targets)
+	if !gopts.JSON {
+		p.V("start backup on %v", targets)
+	}
 	_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)
 	}
 
-	p.Finish()
-	p.P("snapshot %s saved\n", id.Str())
-
 	// cleanly shutdown all running goroutines
 	t.Kill(nil)
 
 	// let's see if one returned an error
 	err = t.Wait()
-	if err != nil {
-		return err
+
+	// Report finished execution
+	p.Finish(id)
+	if !gopts.JSON {
+		p.P("snapshot %s saved\n", id.Str())
 	}
 
-	return nil
+	// Return error if any
+	return err
 }

cmd/restic/cmd_cache.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"sort"
 	"time"
@@ -9,6 +10,7 @@ import (
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
 
@@ -17,6 +19,11 @@ var cmdCache = &cobra.Command{
 	Short: "Operate on local cache directories",
 	Long: `
 The "cache" command allows listing and cleaning local cache directories.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -28,6 +35,7 @@ The "cache" command allows listing and cleaning local cache directories.
 type CacheOptions struct {
 	Cleanup bool
 	MaxAge  uint
+	NoSize  bool
 }
 
 var cacheOptions CacheOptions
@@ -38,6 +46,7 @@ func init() {
 	f := cmdCache.Flags()
 	f.BoolVar(&cacheOptions.Cleanup, "cleanup", false, "remove old cache directories")
 	f.UintVar(&cacheOptions.MaxAge, "max-age", 30, "max age in `days` for cache directories to be considered old")
+	f.BoolVar(&cacheOptions.NoSize, "no-size", false, "do not output the size of the cache directories")
 }
 
 func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
@@ -85,9 +94,22 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 		return nil
 	}
 
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-14s   %-16s %s", "Repository ID", "Last Used", "Old")
-	tab.RowFormat = "%-14s   %-16s %s"
+	tab := table.New()
+
+	type data struct {
+		ID   string
+		Last string
+		Old  string
+		Size string
+	}
+
+	tab.AddColumn("Repo ID", "{{ .ID }}")
+	tab.AddColumn("Last Used", "{{ .Last }}")
+	tab.AddColumn("Old", "{{ .Old }}")
+
+	if !opts.NoSize {
+		tab.AddColumn("Size", "{{ .Size }}")
+	}
 
 	dirs, err := cache.All(cachedir)
 	if err != nil {
@@ -109,14 +131,41 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
 			old = "yes"
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{
+		var size string
+		if !opts.NoSize {
+			bytes, err := dirSize(filepath.Join(cachedir, entry.Name()))
+			if err != nil {
+				return err
+			}
+			size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
+		}
+
+		tab.AddRow(data{
 			entry.Name()[:10],
 			fmt.Sprintf("%d days ago", uint(time.Since(entry.ModTime()).Hours()/24)),
 			old,
+			size,
 		})
 	}
 
 	tab.Write(gopts.stdout)
+	Printf("%d cache dirs in %s\n", len(dirs), cachedir)
 
 	return nil
 }
+
+func dirSize(path string) (int64, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if err != nil || info == nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			size += info.Size()
+		}
+
+		return nil
+	})
+	return size, err
+}

cmd/restic/cmd_cat.go

@@ -18,6 +18,11 @@ var cmdCat = &cobra.Command{
 	Short: "Print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -74,7 +79,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		fmt.Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadAndDecrypt(gopts.ctx, restic.IndexFile, id)
+		buf, err := repo.LoadAndDecrypt(gopts.ctx, nil, restic.IndexFile, id)
 		if err != nil {
 			return err
 		}
@@ -99,7 +104,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return nil
 	case "key":
 		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(gopts.ctx, repo.Backend(), h)
+		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -150,7 +155,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-		buf, err := backend.LoadAll(gopts.ctx, repo.Backend(), h)
+		buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -165,18 +170,15 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 	case "blob":
 		for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
-			list, found := repo.Index().Lookup(id, t)
+			_, found := repo.Index().Lookup(id, t)
 			if !found {
 				continue
 			}
-			blob := list[0]
 
-			buf := make([]byte, blob.Length)
-			n, err := repo.LoadBlob(gopts.ctx, t, id, buf)
+			buf, err := repo.LoadBlob(gopts.ctx, t, id, nil)
 			if err != nil {
 				return err
 			}
-			buf = buf[:n]
 
 			_, err = os.Stdout.Write(buf)
 			return err