Compare commits

..

4 Commits

Author SHA1 Message Date
Leo R. Lundgren
6d39410958 doc: Reword parts of the text, replace rclone with rest-server
The intent here is to make the text more consistent in its use of different
concepts involved in explaining the idea and setup that is explained, and
to make it easier to follow.

We're also replacing rclone with rest-server, not because we dislike rclone
but in order to keep the text to the basic tooling and the main restic eco-
system.

Finally we also remove the previous tip at the end about keeping the SSH tunnel
up, as it will be during the time the SSH session is running (in which the user
is expected to run the restic commands).
2026-02-18 22:19:56 +01:00
JL710
5c3116901e use rclone for rest server instead of docker 2026-01-17 16:29:26 +01:00
JL710
8943ca15ed apply suggestions from Michael Eischer 2025-09-25 13:35:40 +02:00
JL710
a9d51db68d add example for "Pulling a Backup with HTTP over a ssh tunnel" 2025-09-11 16:07:50 +02:00
454 changed files with 7111 additions and 10678 deletions

View File

@@ -1,12 +0,0 @@
# Actual layer caching is impossible due to .git, but
# that must be included for provenance reasons. These ignores
# are strictly for hygenic build.
*
!/*.go
!/go.*
!/cmd/*
!/docker/entrypoint.sh
!/internal/*
!/helpers/*
!/VERSION
!/.git/

View File

@@ -36,7 +36,7 @@ Please always follow these steps:
- Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
-->
- [ ] I have added tests for all code changes, see [writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests)
- [ ] I have added tests for all code changes.
- [ ] I have added documentation for relevant changes (in the manual).
- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
- [ ] I'm done! This pull request is ready for review.

View File

@@ -5,10 +5,6 @@ updates:
directory: "/" # Location of package manifests
schedule:
interval: "monthly"
groups:
golang-x-deps:
patterns:
- "golang.org/x/*"
# Dependencies listed in .github/workflows/*.yml
- package-ecosystem: "github-actions"

View File

@@ -26,10 +26,10 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Log in to the Container registry
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}

View File

@@ -13,7 +13,7 @@ permissions:
contents: read
env:
latest_go: "1.25.x"
latest_go: "1.24.x"
GO111MODULE: on
jobs:
@@ -23,29 +23,29 @@ jobs:
# list of jobs to run:
include:
- job_name: Windows
go: 1.25.x
go: 1.24.x
os: windows-latest
- job_name: macOS
go: 1.25.x
go: 1.24.x
os: macOS-latest
test_fuse: false
- job_name: Linux
go: 1.25.x
go: 1.24.x
os: ubuntu-latest
test_cloud_backends: true
test_fuse: true
check_changelog: true
- job_name: Linux (race)
go: 1.25.x
go: 1.24.x
os: ubuntu-latest
test_fuse: true
test_opts: "-race"
- job_name: Linux
go: 1.24.x
go: 1.23.x
os: ubuntu-latest
test_fuse: true
@@ -57,10 +57,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go ${{ matrix.go }}
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
@@ -220,10 +220,10 @@ jobs:
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version: ${{ env.latest_go }}
@@ -242,18 +242,18 @@ jobs:
checks: write
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v6
uses: actions/setup-go@v5
with:
go-version: ${{ env.latest_go }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v9
uses: golangci/golangci-lint-action@v6
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v2.4.0
version: v1.64.8
args: --verbose --timeout 5m
# only run golangci-lint for pull requests, otherwise ALL hints get
@@ -287,7 +287,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Docker meta
id: meta

View File

@@ -1,95 +1,70 @@
version: "2"
# This is the configuration for golangci-lint for the restic project.
#
# A sample config with all settings is here:
# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
linters:
# only enable the linters listed below
default: none
disable-all: true
enable:
- asciicheck
# ensure that http response bodies are closed
- bodyclose
# restrict imports from other restic packages for internal/backend (cache exempt)
- depguard
- copyloopvar
# make sure all errors returned by functions are handled
- errcheck
# show how code can be simplified
- gosimple
# make sure code is formatted
- gofmt
# examine code and report suspicious constructs, such as Printf calls whose
# arguments do not align with the format string
- govet
# consistent imports
- importas
# detect when assignments to existing variables are not used
- ineffassign
- nolintlint
# make sure names and comments are used according to the conventions
- revive
# detect when assignments to existing variables are not used
- ineffassign
# run static analysis and find errors
- staticcheck
# find unused variables, functions, structs, types, etc.
- unused
settings:
depguard:
rules:
# Prevent backend packages from importing the internal/restic package to keep the architectural layers intact.
backend-imports:
files:
- "**/internal/backend/**"
- "!**/internal/backend/cache/**"
- "!**/internal/backend/test/**"
- "!**/*_test.go"
deny:
- pkg: "github.com/restic/restic/internal/restic"
desc: "internal/restic should not be imported to keep the architectural layers intact"
- pkg: "github.com/restic/restic/internal/repository"
desc: "internal/repository should not be imported to keep the architectural layers intact"
importas:
alias:
- pkg: github.com/restic/restic/internal/test
alias: rtest
staticcheck:
checks:
# default
- "all"
- "-ST1000"
- "-ST1003"
- "-ST1016"
- "-ST1020"
- "-ST1021"
- "-ST1022"
# extra disables
- "-QF1008" # don't warn about specifing name of embedded field on access
exclusions:
rules:
# revive: ignore unused parameters in tests
- path: (_test\.go|testing\.go|backend/.*/tests\.go)
text: "unused-parameter:"
# revive: do not warn about missing comments for exported stuff
- path: (.+)\.go$
text: exported (function|method|var|type|const) .* should have comment or be unexported
# revive: ignore constants in all caps
- path: (.+)\.go$
text: don't use ALL_CAPS in Go names; use CamelCase
# revive: lots of packages don't have such a comment
- path: (.+)\.go$
text: "package-comments: should have a package comment"
# staticcheck: there's no easy way to replace these packages
- path: (.+)\.go$
text: 'SA1019: "golang.org/x/crypto/poly1305" is deprecated'
- path: (.+)\.go$
text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated'
- path: (.+)\.go$
text: "redefines-builtin-id:"
# revive: collection of helpers to implement a backend, more descriptive names would be too repetitive
- path: internal/backend/util/.*.go$
text: "var-naming: avoid meaningless package names"
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
# make sure code is formatted
- gofmt
exclusions:
paths:
- third_party$
- builtin$
- examples$
# parse and typecheck code
- typecheck
# ensure that http response bodies are closed
- bodyclose
- importas
issues:
# don't use the default exclude rules, this hides (among others) ignored
# errors from Close() calls
exclude-use-default: false
# list of things to not warn about
exclude:
# revive: do not warn about missing comments for exported stuff
- exported (function|method|var|type|const) .* should have comment or be unexported
# revive: ignore constants in all caps
- don't use ALL_CAPS in Go names; use CamelCase
# revive: lots of packages don't have such a comment
- "package-comments: should have a package comment"
# staticcheck: there's no easy way to replace these packages
- "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
- "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
- "redefines-builtin-id:"
exclude-rules:
# revive: ignore unused parameters in tests
- path: (_test\.go|testing\.go|backend/.*/tests\.go)
text: "unused-parameter:"
linters-settings:
importas:
alias:
- pkg: github.com/restic/restic/internal/test
alias: rtest

View File

@@ -1,6 +1,5 @@
# Table of Contents
* [Changelog for 0.18.1](#changelog-for-restic-0181-2025-09-21)
* [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
* [Changelog for 0.17.3](#changelog-for-restic-0173-2024-11-08)
* [Changelog for 0.17.2](#changelog-for-restic-0172-2024-10-27)
@@ -40,106 +39,6 @@
* [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
# Changelog for restic 0.18.1 (2025-09-21)
The following sections list the changes in restic 0.18.1 relevant to
restic users. The changes are ordered by importance.
## Summary
* Fix #5324: Correctly handle `backup --stdin-filename` with directory paths
* Fix #5325: Accept `RESTIC_HOST` environment variable in `forget` command
* Fix #5342: Ignore "chmod not supported" errors when writing files
* Fix #5344: Ignore `EOPNOTSUPP` errors for extended attributes
* Fix #5421: Fix rare crash if directory is removed during backup
* Fix #5429: Stop retrying uploads when rest-server runs out of space
* Fix #5467: Improve handling of download retries in `check` command
## Details
* Bugfix #5324: Correctly handle `backup --stdin-filename` with directory paths
In restic 0.18.0, the `backup` command failed if a filename that includes at
least a directory was passed to `--stdin-filename`. For example,
`--stdin-filename /foo/bar` resulted in the following error:
```
Fatal: unable to save snapshot: open /foo: no such file or directory
```
This has now been fixed.
https://github.com/restic/restic/issues/5324
https://github.com/restic/restic/pull/5356
* Bugfix #5325: Accept `RESTIC_HOST` environment variable in `forget` command
The `forget` command did not use the host name from the `RESTIC_HOST`
environment variable when filtering snapshots. This has now been fixed.
https://github.com/restic/restic/issues/5325
https://github.com/restic/restic/pull/5327
* Bugfix #5342: Ignore "chmod not supported" errors when writing files
Restic 0.18.0 introduced a bug that caused `chmod xxx: operation not supported`
errors to appear when writing to a local file repository that did not support
chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors.
https://github.com/restic/restic/issues/5342
* Bugfix #5344: Ignore `EOPNOTSUPP` errors for extended attributes
Restic 0.18.0 added extended attribute support for NetBSD 10+, but not all
NetBSD filesystems support extended attributes. Other BSD systems can likewise
return `EOPNOTSUPP`, so restic now ignores these errors.
https://github.com/restic/restic/issues/5344
* Bugfix #5421: Fix rare crash if directory is removed during backup
In restic 0.18.0, the `backup` command could crash if a directory was removed
between reading its metadata and listing its directory content. This has now
been fixed.
https://github.com/restic/restic/pull/5421
* Bugfix #5429: Stop retrying uploads when rest-server runs out of space
When rest-server returns a `507 Insufficient Storage` error, it indicates that
no more storage capacity is available. Restic now correctly stops retrying
uploads in this case.
https://github.com/restic/restic/issues/5429
https://github.com/restic/restic/pull/5452
* Bugfix #5467: Improve handling of download retries in `check` command
In very rare cases, the `check` command could unnecessarily report repository
damage if the backend returned incomplete, corrupted data on the first download
try which is afterwards resolved by a download retry.
This could result in an error output like the following:
```
Load(<data/34567890ab>, 33918928, 0) returned error, retrying after 871.35598ms: readFull: unexpected EOF
Load(<data/34567890ab>, 33918928, 0) operation successful after 1 retries
check successful on second attempt, original error pack 34567890ab[...] contains 6 errors: [blob 12345678[...]: decrypting blob <data/12345678> from 34567890 failed: ciphertext verification failed ...]
[...]
Fatal: repository contains errors
```
This fix only applies to a very specific case where the log shows `operation
successful after 1 retries` followed by a `check successful on second attempt,
original error` that only reports `ciphertext verification failed` errors in the
pack file. If any other errors are reported in the pack file, then the
repository still has to be considered as damaged.
Now, only the check result of the last download retry is reported as intended.
https://github.com/restic/restic/issues/5467
https://github.com/restic/restic/pull/5495
# Changelog for restic 0.18.0 (2025-03-27)
The following sections list the changes in restic 0.18.0 relevant to
restic users. The changes are ordered by importance.

View File

@@ -202,9 +202,6 @@ we'll be glad to assist. Having a PR with failing integration tests is nothing
to be ashamed of. In contrast, that happens regularly for all of us. That's
what the tests are there for.
More details of how to structure tests can be found here at
[writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests).
Git Commits
-----------

View File

@@ -1 +1 @@
0.18.1-dev
0.18.0-dev

View File

@@ -36,6 +36,7 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//go:build ignore_build_go
// +build ignore_build_go
package main
@@ -59,7 +60,7 @@ var config = Config{
// see https://github.com/googleapis/google-cloud-go/issues/11448
DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run
MinVersion: GoVersion{Major: 1, Minor: 24, Patch: 0}, // minimum Go version supported
MinVersion: GoVersion{Major: 1, Minor: 23, Patch: 0}, // minimum Go version supported
}
// Config configures the build.

View File

@@ -1,7 +0,0 @@
Bugfix: Ignore `EOPNOTSUPP` errors for extended attributes
Restic 0.18.0 added extended attribute support for NetBSD 10+, but not all
NetBSD filesystems support extended attributes. Other BSD systems can
likewise return `EOPNOTSUPP`, so restic now ignores these errors.
https://github.com/restic/restic/issues/5344

View File

@@ -1,8 +0,0 @@
Bugfix: Stop retrying uploads when rest-server runs out of space
When rest-server returns a `507 Insufficient Storage` error, it indicates
that no more storage capacity is available. Restic now correctly stops
retrying uploads in this case.
https://github.com/restic/restic/issues/5429
https://github.com/restic/restic/pull/5452

View File

@@ -1,27 +0,0 @@
Bugfix: Improve handling of download retries in `check` command
In very rare cases, the `check` command could unnecessarily report repository
damage if the backend returned incomplete, corrupted data on the first download
try which is afterwards resolved by a download retry.
This could result in an error output like the following:
```
Load(<data/34567890ab>, 33918928, 0) returned error, retrying after 871.35598ms: readFull: unexpected EOF
Load(<data/34567890ab>, 33918928, 0) operation successful after 1 retries
check successful on second attempt, original error pack 34567890ab[...] contains 6 errors: [blob 12345678[...]: decrypting blob <data/12345678> from 34567890 failed: ciphertext verification failed ...]
[...]
Fatal: repository contains errors
```
This fix only applies to a very specific case where the log shows
`operation successful after 1 retries` followed by a
`check successful on second attempt, original error` that only reports
`ciphertext verification failed` errors in the pack file. If any other errors
are reported in the pack file, then the repository still has to be considered
as damaged.
Now, only the check result of the last download retry is reported as intended.
https://github.com/restic/restic/issues/5467
https://github.com/restic/restic/pull/5495

View File

@@ -1,9 +0,0 @@
Enhancement: `restic check` for specified snapshot(s) via snapshot filtering
Snapshots can now be specified for the command `restic check` on the command line
via the standard snapshot filter, (`--tag`, `--host`, `--path` or specifying
snapshot IDs directly) and will be used for checking the packfiles used by these snapshots.
https://github.com/restic/restic/issues/3326
https://github.com/restic/restic/pull/5469
https://github.com/restic/restic/pull/5644

View File

@@ -1,9 +0,0 @@
Enhancement: Support restoring ownership by name on UNIX systems
Restic restore used to restore file ownership on UNIX systems by UID and GID.
It now allows restoring the file ownership by user name and group name with `--ownership-by-name`.
This allows restoring snapshots on a system where the UID/GID are not the same as they were on the system where the snapshot was created.
However it does not include support for POSIX ACLs, which are still restored by their numeric value.
https://github.com/restic/restic/issues/3572
https://github.com/restic/restic/pull/5449

View File

@@ -1,8 +0,0 @@
Enhancement: Allow Github personal access token to be specified for `self-update`
`restic self-update` previously only used unauthenticated GitHub API requests when checking for the latest release. This caused some users sharing IP addresses to hit the GitHub rate limit, resulting in a 403 Forbidden error and preventing updates.
Restic still uses unauthenticated requests by default, but it now optionally supports authenticated GitHub API requests during `self-update`. Users can set the `$GITHUB_ACCESS_TOKEN` environment variable to use a [personal access token](https://github.com/settings/tokens) for this effect, avoiding update failures due to rate limiting.
https://github.com/restic/restic/issues/3738
https://github.com/restic/restic/pull/5568

View File

@@ -1,12 +0,0 @@
Enhancement: Support include filters in `rewrite` command
The enhancement enables the standard include filter options
--iinclude pattern same as --include pattern but ignores the casing of filenames
--iinclude-file file same as --include-file but ignores casing of filenames in patterns
-i, --include pattern include a pattern (can be specified multiple times)
--include-file file read include patterns from a file (can be specified multiple times)
The exclusion or inclusion of filter parameters is exclusive, as in other commands.
https://github.com/restic/restic/issues/4278
https://github.com/restic/restic/pull/5191

View File

@@ -1,11 +0,0 @@
Bugfix: Exit with code 3 when some `backup` source files do not exist
Restic used to exit with code 0 even when some backup sources did not exist. Restic
would exit with code 3 only when child directories or files did not exist. This
could cause confusion and unexpected behavior in scripts that relied on the exit
code to determine if the backup was successful.
Restic now exits with code 3 when some backup sources do not exist.
https://github.com/restic/restic/issues/4467
https://github.com/restic/restic/pull/5347

View File

@@ -1,7 +0,0 @@
Bugfix: Exit with correct code on SIGINT
Restic previously returned exit code 1 on SIGINT, which is incorrect.
Restic now returns 130 on SIGINT.
https://github.com/restic/restic/issues/5258
https://github.com/restic/restic/pull/5363

View File

@@ -1,7 +0,0 @@
Bugfix: `restic find` now checks for correct ordering of time related options
`restic find` now immediately fails with an error if both `--oldest` and `--newest` are specified
and `--oldest` is a timestamp after `--newest`.
https://github.com/restic/restic/issues/5280
https://github.com/restic/restic/pull/5310

View File

@@ -1,14 +1,14 @@
Bugfix: Correctly handle `backup --stdin-filename` with directory paths
Bugfix: Correctly handle `backup --stdin-filename` with directories
In restic 0.18.0, the `backup` command failed if a filename that includes
at least a directory was passed to `--stdin-filename`. For example,
a least a directory was passed to `--stdin-filename`. For example,
`--stdin-filename /foo/bar` resulted in the following error:
```
Fatal: unable to save snapshot: open /foo: no such file or directory
```
This has now been fixed.
This has been fixed now.
https://github.com/restic/restic/issues/5324
https://github.com/restic/restic/pull/5356

View File

@@ -1,7 +1,7 @@
Bugfix: Accept `RESTIC_HOST` environment variable in `forget` command
Bugfix: Correctly handle `RESTIC_HOST` in `forget` command
The `forget` command did not use the host name from the `RESTIC_HOST`
environment variable when filtering snapshots. This has now been fixed.
environment variable. This has been fixed.
https://github.com/restic/restic/issues/5325
https://github.com/restic/restic/pull/5327

View File

@@ -1,6 +1,6 @@
Bugfix: Ignore "chmod not supported" errors when writing files
Restic 0.18.0 introduced a bug that caused `chmod xxx: operation not supported`
Restic 0.18.0 introduced a bug that caused "chmod xxx: operation not supported"
errors to appear when writing to a local file repository that did not support
chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors.

View File

@@ -0,0 +1,7 @@
Bugfix: Ignore EOPNOTSUPP as an error for xattr
Restic 0.18.0 added xattr support for NetBSD 10+, but not all NetBSD
filesystems support xattrs. Other BSD systems can likewise return
EOPNOTSUPP, so restic now simply ignores EOPNOTSUPP errors for xattrs.
https://github.com/restic/restic/issues/5344

View File

@@ -1,11 +0,0 @@
Enhancement: Add support for --exclude-cloud-files on macOS (e.g. iCloud drive)
Restic treated files stored in iCloud drive as though they were regular files.
This caused restic to download all files (including files marked as cloud only) while iterating over them.
Restic now allows the user to exclude these files when backing up with the `--exclude-cloud-files` option.
Works from Sonoma (macOS 14.0) onwards. Older macOS versions materialize files when `stat` is called on the file.
https://github.com/restic/restic/pull/4990
https://github.com/restic/restic/issues/5352

View File

@@ -10,5 +10,3 @@ This has been fixed.
https://github.com/restic/restic/issues/5354
https://github.com/restic/restic/pull/5358
https://github.com/restic/restic/pull/5493
https://github.com/restic/restic/pull/5494

View File

@@ -1,10 +0,0 @@
Enhancement: Reduce progress bar refresh rates to reduce energy usage
Progress bars were updated with 60fps which can cause high CPU or GPU usage
for some terminal emulators. Reduce it to 10fps to conserve energy.
In addition, this lower frequency seem to be necessary to allow selecting
anything in the terminal with certain terminal emulators.
https://github.com/restic/restic/issues/5383
https://github.com/restic/restic/pull/5551
https://github.com/restic/restic/pull/5626

View File

@@ -0,0 +1,8 @@
Bugfix: do not retry if rest-server runs out of space
Rest-server return error `507 Insufficient Storage` if no more storage
capacity is available at the server. Restic now no longer retries uploads
in this case.
https://github.com/restic/restic/issues/5429
https://github.com/restic/restic/pull/5452

View File

@@ -1,12 +0,0 @@
Enhancement: Allow overriding RESTIC_HOST environment variable with --host flag
When the `RESTIC_HOST` environment variable was set, there was no way to list or
operate on snapshots from all hosts, as the environment variable would always
filter to that specific host. Restic now allows overriding `RESTIC_HOST` by
explicitly providing the `--host` flag with an empty string (e.g., `--host=""` or
`--host=`), which will show snapshots from all hosts. This works for all commands
that support snapshot filtering: `snapshots`, `forget`, `find`, `stats`, `copy`,
`tag`, `repair snapshots`, `rewrite`, `mount`, `restore`, `dump`, and `ls`.
https://github.com/restic/restic/issues/5440
https://github.com/restic/restic/pull/5541

View File

@@ -1,10 +0,0 @@
Enhancement: `copy` copies snapshots in batches
The `copy` command used to copy snapshots individually, even if this resulted in creating pack files
smaller than the target pack size. In particular, this resulted in many small files
when copying small incremental snapshots.
Now, `copy` copies multiple snapshots at once to avoid creating small files.
https://github.com/restic/restic/issues/5175
https://github.com/restic/restic/pull/5464

View File

@@ -1,7 +0,0 @@
Bugfix: Password prompt was sometimes not shown
The password prompt for a repository was sometimes not shown when running
the `backup -v` command. This has been fixed.
https://github.com/restic/restic/issues/5477
https://github.com/restic/restic/pull/5554

View File

@@ -1,8 +0,0 @@
Bugfix: Mark files as readonly when using the SFTP backend
Files created by the SFTP backend previously allowed writes to those files.
Restic now restricts the file permissions on SFTP backend to readonly.
This change only has an effect for sftp servers with support for the chmod operation.
https://github.com/restic/restic/issues/5487
https://github.com/restic/restic/pull/5497

View File

@@ -1,15 +0,0 @@
Enhancement: Reduce Azure storage costs by optimizing upload method
Restic previously used Azure's PutBlock and PutBlockList APIs for all file
uploads, which resulted in two transactions per file and doubled the storage
operation costs. For backups with many pack files, this could lead to
significant Azure storage transaction fees.
Restic now uses the more efficient PutBlob API for files up to 256 MiB,
requiring only a single transaction per file. This reduces Azure storage
operation costs by approximately 50% for typical backup workloads. Files
larger than 256 MiB continue to use the block-based upload method as required
by Azure's API limits.
https://github.com/restic/restic/issues/5531
https://github.com/restic/restic/pull/5544

View File

@@ -1,7 +0,0 @@
Bugfix: correctly handle `snapshots --group-by` in combination with `--latest`
For the `snapshots` command, the `--latest` option did not correctly handle the
case where an non-default value was passed to `--group-by`. This has been fixed.
https://github.com/restic/restic/issues/5586
https://github.com/restic/restic/pull/5601

View File

@@ -1,8 +0,0 @@
Bugfix: Fix "chmod not supported" errors when unlocking
Restic 0.18.0 introduced a bug that caused "chmod xxx: operation not supported"
errors to appear when unlocking with a stale lock, on a local file repository
that did not support chmod (like CIFS or WebDAV mounted via FUSE). Restic now
just doesn't bother calling chmod in that case on Unix, as it is unnecessary.
https://github.com/restic/restic/issues/5595

View File

@@ -1,5 +0,0 @@
Change: Update dependencies and require Go 1.24 or newer
We have updated all dependencies. Restic now requires Go 1.24 or newer to build.
https://github.com/restic/restic/pull/5619

View File

@@ -1,9 +0,0 @@
Enhancement: add more status counters to `restic copy`
`restic copy` now produces more status counters in text format. The new counters
are the number of blobs to copy, their size on disk and the number of packfiles
used from the source repository. The additional statistics is only produced when
the `--verbose` option is specified.
https://github.com/restic/restic/issues/5175
https://github.com/restic/restic/pull/5319

View File

@@ -1,7 +1,8 @@
Bugfix: Fix rare crash if directory is removed during backup
In restic 0.18.0, the `backup` command could crash if a directory was removed
between reading its metadata and listing its directory content. This has now
been fixed.
In restic 0.18.0, the `backup` command could crash if a directory is removed
inbetween reading its metadata and listing its directory content.
This has been fixed.
https://github.com/restic/restic/pull/5421

View File

@@ -1,11 +0,0 @@
Enhancement: Enable file system privileges on Windows before access
Restic attempted to enable Windows file system privileges when
reading or writing security descriptors - after potentially being wholly
denied access to previous items. It also read file extended attributes without
using the privilege, possibly missing them and producing errors.
Restic now attempts to enable all file system privileges before any file
access. It also requests extended attribute reads use the backup privilege.
https://github.com/restic/restic/pull/5424

View File

@@ -1,11 +0,0 @@
Enhancement: Allow nice and ionice configuration for restic containers
The official restic docker now supports the following environment variables:
`NICE`: set the desired nice scheduling. See `man nice`.
`IONICE_CLASS`: set the desired I/O scheduling class. See `man ionice`. Note that real time support requires the invoker to manually add the `SYS_NICE` capability.
`IONICE_PRIORITY`: set the prioritization for ionice in the given `IONICE_CLASS`. This does nothing without `IONICE_CLASS`, but defaults to `4` (no priority, no penalties).
See https://restic.readthedocs.io/en/stable/020_installation.html#docker-container for further details.
https://github.com/restic/restic/pull/5448

View File

@@ -1,10 +0,0 @@
Bugfix: Correctly restore ACL inheritance state on Windows
Since the introduction of Security Descriptor backups in restic 0.17.0, the inheritance property of Access Control Entries (ACEs) was not restored correctly. This resulted in all restored permissions being marked as explicit (IsInherited: False), even if they were originally inherited from a parent folder.
The issue was caused by sending conflicting inheritance flags (PROTECTED_... and UNPROTECTED_...) to the Windows API during the restore process. The API would default to the more restrictive PROTECTED state, effectively disabling inheritance.
This has been fixed by ensuring that only the correct, non-conflicting inheritance flag is used when applying the security descriptor, preserving the original permission structure from the backup.
https://github.com/restic/restic/pull/5465
https://github.com/restic/restic/issues/5427

View File

@@ -1,6 +0,0 @@
Enhancement: Add OpenContainers labels to Dockerfile.release
The restic Docker image now includes labels from the OpenContainers Annotations Spec.
This information can be used by third party services.
https://github.com/restic/restic/pull/5523

View File

@@ -1,10 +0,0 @@
Enhancement: Display timezone information in snapshots output
The `snapshots` command now displays which timezone is being used to show
timestamps. Since snapshots can be created in different timezones but are
always displayed in the local timezone, a footer line is now shown indicating
the timezone used for display (e.g., "Timestamps shown in CET timezone").
This helps prevent confusion when comparing snapshots in a multi-user
environment.
https://github.com/restic/restic/pull/5588

View File

@@ -1,7 +0,0 @@
Bugfix: Return error if `RESTIC_PACK_SIZE` contains invalid value
If the environment variable `RESTIC_PACK_SIZE` could not be parsed, then
restic ignored its value. Now, the restic commands fail with an error, unless
the command-line option `--pack-size` was specified.
https://github.com/restic/restic/pull/5592

View File

@@ -1,7 +0,0 @@
Enhancement: reduce memory usage of check/copy/diff/stats commands
We have optimized the memory usage of the `check`, `copy`, `diff` and
`stats` commands. These now require less memory when processing large
snapshots.
https://github.com/restic/restic/pull/5610

View File

@@ -1,9 +0,0 @@
Enhancement: stricter early mountpoint validation in `mount`
`restic mount` accepted parameters that would lead to a FUSE mount operation
failing after having done computationally intensive work to prepare the mount.
The `mountpoint` argument supplied must now refer to the name of a directory
that the current user can access and write to, otherwise `restic mount` will
exit with an error before interacting with the repository.
https://github.com/restic/restic/pull/5718

View File

@@ -2,8 +2,6 @@ package main
import (
"context"
"fmt"
"io"
"os"
"os/signal"
"syscall"
@@ -11,27 +9,26 @@ import (
"github.com/restic/restic/internal/debug"
)
func createGlobalContext(stderr io.Writer) context.Context {
func createGlobalContext() context.Context {
ctx, cancel := context.WithCancel(context.Background())
ch := make(chan os.Signal, 1)
go cleanupHandler(ch, cancel, stderr)
go cleanupHandler(ch, cancel)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
return ctx
}
// cleanupHandler handles the SIGINT and SIGTERM signals.
func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc, stderr io.Writer) {
func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
s := <-c
debug.Log("signal %v received, cleaning up", s)
// ignore error as there's no good way to handle it
_, _ = fmt.Fprintf(stderr, "\rsignal %v received, cleaning up \n", s)
Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
_, _ = stderr.Write([]byte("\n--- STACKTRACE START ---\n\n"))
_, _ = stderr.Write([]byte(debug.DumpStacktrace()))
_, _ = stderr.Write([]byte("\n--- STACKTRACE END ---\n"))
_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
}
cancel()

View File

@@ -19,20 +19,19 @@ import (
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/textfile"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/backup"
"github.com/restic/restic/internal/ui/termstatus"
)
func newBackupCommand(globalOptions *global.Options) *cobra.Command {
func newBackupCommand() *cobra.Command {
var opts BackupOptions
cmd := &cobra.Command{
@@ -65,7 +64,9 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runBackup(cmd.Context(), opts, *globalOptions, globalOptions.Term, args)
term, cancel := setupTermstatus()
defer cancel()
return runBackup(cmd.Context(), opts, globalOptions, term, args)
},
}
@@ -78,7 +79,7 @@ type BackupOptions struct {
filter.ExcludePatternOptions
Parent string
GroupBy data.SnapshotGroupByOptions
GroupBy restic.SnapshotGroupByOptions
Force bool
ExcludeOtherFS bool
ExcludeIfPresent []string
@@ -88,7 +89,7 @@ type BackupOptions struct {
Stdin bool
StdinFilename string
StdinCommand bool
Tags data.TagLists
Tags restic.TagLists
Host string
FilesFrom []string
FilesFromVerbatim []string
@@ -106,7 +107,7 @@ type BackupOptions struct {
func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) {
f.StringVar(&opts.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true}
opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
f.BoolVarP(&opts.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
@@ -139,9 +140,7 @@ func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
if runtime.GOOS == "windows" {
f.BoolVar(&opts.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
}
if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive, iCloud drive, …)")
f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive Files On-Demand)")
}
f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
@@ -160,16 +159,13 @@ var backupFSTestHook func(fs fs.FS) fs.FS
// ErrInvalidSourceData is used to report an incomplete backup
var ErrInvalidSourceData = errors.New("at least one source file could not be read")
// ErrNoSourceData is used to report that no source data was found
var ErrNoSourceData = errors.Fatal("all source directories/files do not exist")
// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string, warnf func(msg string, args ...interface{})) (result []string, err error) {
func filterExisting(items []string) (result []string, err error) {
for _, item := range items {
_, err := fs.Lstat(item)
if errors.Is(err, os.ErrNotExist) {
warnf("%v does not exist, skipping\n", item)
Warnf("%v does not exist, skipping\n", item)
continue
}
@@ -177,12 +173,10 @@ func filterExisting(items []string, warnf func(msg string, args ...interface{}))
}
if len(result) == 0 {
return nil, ErrNoSourceData
} else if len(result) < len(items) {
return result, ErrInvalidSourceData
return nil, errors.Fatal("all source directories/files do not exist")
}
return result, nil
return
}
// readLines reads all lines from the named file and returns them as a
@@ -191,7 +185,7 @@ func filterExisting(items []string, warnf func(msg string, args ...interface{}))
// If filename is empty, readPatternsFromFile returns an empty slice.
// If filename is a dash (-), readPatternsFromFile will read the lines from the
// standard input.
func readLines(filename string, stdin io.ReadCloser) ([]string, error) {
func readLines(filename string) ([]string, error) {
if filename == "" {
return nil, nil
}
@@ -202,7 +196,7 @@ func readLines(filename string, stdin io.ReadCloser) ([]string, error) {
)
if filename == "-" {
data, err = io.ReadAll(stdin)
data, err = io.ReadAll(os.Stdin)
} else {
data, err = textfile.Read(filename)
}
@@ -227,8 +221,8 @@ func readLines(filename string, stdin io.ReadCloser) ([]string, error) {
// readFilenamesFromFileRaw reads a list of filenames from the given file,
// or stdin if filename is "-". Each filename is terminated by a zero byte,
// which is stripped off.
func readFilenamesFromFileRaw(filename string, stdin io.ReadCloser) (names []string, err error) {
f := stdin
func readFilenamesFromFileRaw(filename string) (names []string, err error) {
f := os.Stdin
if filename != "-" {
if f, err = os.Open(filename); err != nil {
return nil, err
@@ -277,8 +271,8 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
}
// Check returns an error when an invalid combination of options was set.
func (opts BackupOptions) Check(gopts global.Options, args []string) error {
if gopts.Password == "" && !gopts.InsecureNoPassword {
func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
if gopts.password == "" && !gopts.InsecureNoPassword {
if opts.Stdin {
return errors.Fatal("cannot read both password and data from stdin")
}
@@ -312,7 +306,7 @@ func (opts BackupOptions) Check(gopts global.Options, args []string) error {
// collectRejectByNameFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path only
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, warnf func(msg string, args ...interface{})) (fs []archiver.RejectByNameFunc, err error) {
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) {
// exclude restic cache
if repo.Cache() != nil {
f, err := rejectResticCache(repo)
@@ -323,7 +317,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, w
fs = append(fs, f)
}
fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(warnf)
fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
if err != nil {
return nil, err
}
@@ -336,7 +330,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, w
// collectRejectFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path and file info
func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf func(msg string, args ...interface{})) (funcs []archiver.RejectFunc, err error) {
func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS) (funcs []archiver.RejectFunc, err error) {
// allowed devices
if opts.ExcludeOtherFS && !opts.Stdin && !opts.StdinCommand {
f, err := archiver.RejectByDevice(targets, fs)
@@ -360,7 +354,10 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu
}
if opts.ExcludeCloudFiles && !opts.Stdin && !opts.StdinCommand {
f, err := archiver.RejectCloudFiles(warnf)
if runtime.GOOS != "windows" {
return nil, errors.Fatalf("exclude-cloud-files is only supported on Windows")
}
f, err := archiver.RejectCloudFiles(Warnf)
if err != nil {
return nil, err
}
@@ -372,7 +369,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu
}
for _, spec := range opts.ExcludeIfPresent {
f, err := archiver.RejectIfPresent(spec, warnf)
f, err := archiver.RejectIfPresent(spec, Warnf)
if err != nil {
return nil, err
}
@@ -384,13 +381,13 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu
}
// collectTargets returns a list of target files/dirs from several sources.
func collectTargets(opts BackupOptions, args []string, warnf func(msg string, args ...interface{}), stdin io.ReadCloser) (targets []string, err error) {
func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
if opts.Stdin || opts.StdinCommand {
return nil, nil
}
for _, file := range opts.FilesFrom {
fromfile, err := readLines(file, stdin)
fromfile, err := readLines(file)
if err != nil {
return nil, err
}
@@ -408,14 +405,14 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
return nil, fmt.Errorf("pattern: %s: %w", line, err)
}
if len(expanded) == 0 {
warnf("pattern %q does not match any files, skipping\n", line)
Warnf("pattern %q does not match any files, skipping\n", line)
}
targets = append(targets, expanded...)
}
}
for _, file := range opts.FilesFromVerbatim {
fromfile, err := readLines(file, stdin)
fromfile, err := readLines(file)
if err != nil {
return nil, err
}
@@ -428,7 +425,7 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
}
for _, file := range opts.FilesFromRaw {
fromfile, err := readFilenamesFromFileRaw(file, stdin)
fromfile, err := readFilenamesFromFileRaw(file)
if err != nil {
return nil, err
}
@@ -442,12 +439,17 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
return nil, errors.Fatal("nothing to backup, please specify source files/dirs")
}
return filterExisting(targets, warnf)
targets, err = filterExisting(targets)
if err != nil {
return nil, err
}
return targets, nil
}
// parent returns the ID of the parent snapshot. If there is none, nil is
// returned.
func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*data.Snapshot, error) {
func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
if opts.Force {
return nil, nil
}
@@ -456,7 +458,7 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o
if snName == "" {
snName = "latest"
}
f := data.SnapshotFilter{TimestampLimit: timeStampLimit}
f := restic.SnapshotFilter{TimestampLimit: timeStampLimit}
if opts.GroupBy.Host {
f.Hosts = []string{opts.Host}
}
@@ -464,29 +466,23 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o
f.Paths = targets
}
if opts.GroupBy.Tag {
f.Tags = []data.TagList{opts.Tags.Flatten()}
f.Tags = []restic.TagList{opts.Tags.Flatten()}
}
sn, _, err := f.FindLatest(ctx, repo, repo, snName)
// Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, data.ErrNoSnapshotFound) {
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil
}
return sn, err
}
func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, term ui.Terminal, args []string) error {
func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
var vsscfg fs.VSSConfig
var err error
var printer backup.ProgressPrinter
if gopts.JSON {
printer = backup.NewJSONProgress(term, gopts.Verbosity)
} else {
printer = backup.NewTextProgress(term, gopts.Verbosity)
}
if runtime.GOOS == "windows" {
if vsscfg, err = fs.ParseVSSConfig(gopts.Extended); err != nil {
if vsscfg, err = fs.ParseVSSConfig(gopts.extended); err != nil {
return err
}
}
@@ -496,46 +492,47 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
return err
}
success := true
targets, err := collectTargets(opts, args, printer.E, term.InputRaw())
targets, err := collectTargets(opts, args)
if err != nil {
if errors.Is(err, ErrInvalidSourceData) {
success = false
} else {
return err
}
return err
}
timeStamp := time.Now()
backupStart := timeStamp
if opts.TimeStamp != "" {
timeStamp, err = time.ParseInLocation(global.TimeFormat, opts.TimeStamp, time.Local)
timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
if err != nil {
return errors.Fatalf("error in time option: %v", err)
return errors.Fatalf("error in time option: %v\n", err)
}
}
if gopts.Verbosity >= 2 && !gopts.JSON {
printer.P("open repository")
if gopts.verbosity >= 2 && !gopts.JSON {
Verbosef("open repository\n")
}
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun, printer)
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun)
if err != nil {
return err
}
defer unlock()
progressReporter := backup.NewProgress(printer,
ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()))
var progressPrinter backup.ProgressPrinter
if gopts.JSON {
progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
} else {
progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
}
progressReporter := backup.NewProgress(progressPrinter,
calculateProgressInterval(!gopts.Quiet, gopts.JSON))
defer progressReporter.Done()
// rejectByNameFuncs collect functions that can reject items from the backup based on path only
rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, printer.E)
rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
if err != nil {
return err
}
var parentSnapshot *data.Snapshot
var parentSnapshot *restic.Snapshot
if !opts.Stdin {
parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
if err != nil {
@@ -544,18 +541,19 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
if !gopts.JSON {
if parentSnapshot != nil {
printer.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
progressPrinter.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
} else {
printer.P("no parent snapshot found, will read all files\n")
progressPrinter.P("no parent snapshot found, will read all files\n")
}
}
}
if !gopts.JSON {
printer.V("load index files")
progressPrinter.V("load index files")
}
err = repo.LoadIndex(ctx, printer)
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
@@ -572,7 +570,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
messageHandler := func(msg string, args ...interface{}) {
if !gopts.JSON {
printer.P(msg, args...)
progressPrinter.P(msg, args...)
}
}
@@ -583,12 +581,12 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
if opts.Stdin || opts.StdinCommand {
if !gopts.JSON {
printer.V("read data from stdin")
progressPrinter.V("read data from stdin")
}
filename := path.Join("/", opts.StdinFilename)
source := term.InputRaw()
var source io.ReadCloser = os.Stdin
if opts.StdinCommand {
source, err = fs.NewCommandReader(ctx, args, printer.E)
source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
if err != nil {
return err
}
@@ -608,7 +606,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
}
// rejectFuncs collect functions that can reject items from the backup based on path and file info
rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS, printer.E)
rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS)
if err != nil {
return err
}
@@ -624,11 +622,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
sc := archiver.NewScanner(targetFS)
sc.SelectByName = selectByNameFilter
sc.Select = selectFilter
sc.Error = printer.ScannerError
sc.Error = progressPrinter.ScannerError
sc.Result = progressReporter.ReportTotal
if !gopts.JSON {
printer.V("start scan on %v", targets)
progressPrinter.V("start scan on %v", targets)
}
wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
}
@@ -637,7 +635,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
arch.SelectByName = selectByNameFilter
arch.Select = selectFilter
arch.WithAtime = opts.WithAtime
success := true
arch.Error = func(item string, err error) error {
success = false
reterr := progressReporter.Error(item, err)
@@ -668,12 +666,12 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
Time: timeStamp,
Hostname: opts.Host,
ParentSnapshot: parentSnapshot,
ProgramVersion: "restic " + global.Version,
ProgramVersion: "restic " + version,
SkipIfUnchanged: opts.SkipIfUnchanged,
}
if !gopts.JSON {
printer.V("start backup on %v", targets)
progressPrinter.V("start backup on %v", targets)
}
_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)

View File

@@ -3,34 +3,33 @@ package main
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) error {
return withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
t.Logf("backing up %v in %v", target, dir)
if dir != "" {
cleanup := rtest.Chdir(t, dir)
defer cleanup()
}
opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true}
return runBackup(ctx, opts, gopts, gopts.Term, target)
opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
return runBackup(ctx, opts, gopts, term, target)
})
}
func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) {
func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
rtest.Assert(t, err == nil, "Error while backing up: %v", err)
}
@@ -57,13 +56,13 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
testListSnapshots(t, env.gopts, 1)
testRunCheck(t, env.gopts)
stat1 := dirStats(t, env.repo)
stat1 := dirStats(env.repo)
// second backup, implicit incremental
testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
snapshotIDs := testListSnapshots(t, env.gopts, 2)
stat2 := dirStats(t, env.repo)
stat2 := dirStats(env.repo)
if stat2.size > stat1.size+stat1.size/10 {
t.Error("repository size has grown by more than 10 percent")
}
@@ -75,7 +74,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
snapshotIDs = testListSnapshots(t, env.gopts, 3)
stat3 := dirStats(t, env.repo)
stat3 := dirStats(env.repo)
if stat3.size > stat1.size+stat1.size/10 {
t.Error("repository size has grown by more than 10 percent")
}
@@ -86,7 +85,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
testRunRestore(t, env.gopts, restoredir, snapshotID.String()+":"+toPathInSnapshot(filepath.Dir(env.testdata)))
diff := directoriesContentsDiff(t, env.testdata, filepath.Join(restoredir, "testdata"))
diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
rtest.Assert(t, diff == "", "directories are not equal: %v", diff)
}
@@ -219,41 +218,41 @@ func TestDryRunBackup(t *testing.T) {
// dry run before first backup
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDs := testListSnapshots(t, env.gopts, 0)
packIDs := testRunList(t, env.gopts, "packs")
packIDs := testRunList(t, "packs", env.gopts)
rtest.Assert(t, len(packIDs) == 0,
"expected no data, got %v", snapshotIDs)
indexIDs := testRunList(t, env.gopts, "index")
indexIDs := testRunList(t, "index", env.gopts)
rtest.Assert(t, len(indexIDs) == 0,
"expected no index, got %v", snapshotIDs)
// first backup
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshotIDs = testListSnapshots(t, env.gopts, 1)
packIDs = testRunList(t, env.gopts, "packs")
indexIDs = testRunList(t, env.gopts, "index")
packIDs = testRunList(t, "packs", env.gopts)
indexIDs = testRunList(t, "index", env.gopts)
// dry run between backups
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDsAfter := testListSnapshots(t, env.gopts, 1)
rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
dataIDsAfter := testRunList(t, env.gopts, "packs")
dataIDsAfter := testRunList(t, "packs", env.gopts)
rtest.Equals(t, packIDs, dataIDsAfter)
indexIDsAfter := testRunList(t, env.gopts, "index")
indexIDsAfter := testRunList(t, "index", env.gopts)
rtest.Equals(t, indexIDs, indexIDsAfter)
// second backup, implicit incremental
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshotIDs = testListSnapshots(t, env.gopts, 2)
packIDs = testRunList(t, env.gopts, "packs")
indexIDs = testRunList(t, env.gopts, "index")
packIDs = testRunList(t, "packs", env.gopts)
indexIDs = testRunList(t, "index", env.gopts)
// another dry run
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDsAfter = testListSnapshots(t, env.gopts, 2)
rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
dataIDsAfter = testRunList(t, env.gopts, "packs")
dataIDsAfter = testRunList(t, "packs", env.gopts)
rtest.Equals(t, packIDs, dataIDsAfter)
indexIDsAfter = testRunList(t, env.gopts, "index")
indexIDsAfter = testRunList(t, "index", env.gopts)
rtest.Equals(t, indexIDs, indexIDsAfter)
}
@@ -263,27 +262,22 @@ func TestBackupNonExistingFile(t *testing.T) {
testSetupBackupData(t, env)
p := filepath.Join(env.testdata, "0", "0", "9")
dirs := []string{
filepath.Join(p, "0"),
filepath.Join(p, "1"),
filepath.Join(p, "nonexisting"),
filepath.Join(p, "5"),
}
_ = withRestoreGlobalOptions(func() error {
globalOptions.stderr = io.Discard
opts := BackupOptions{}
p := filepath.Join(env.testdata, "0", "0", "9")
dirs := []string{
filepath.Join(p, "0"),
filepath.Join(p, "1"),
filepath.Join(p, "nonexisting"),
filepath.Join(p, "5"),
}
// mix of existing and non-existing files
err := testRunBackupAssumeFailure(t, "", dirs, opts, env.gopts)
rtest.Assert(t, err != nil, "expected error for non-existing file")
rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "expected ErrInvalidSourceData; got %v", err)
// only non-existing file
dirs = []string{
filepath.Join(p, "nonexisting"),
}
err = testRunBackupAssumeFailure(t, "", dirs, opts, env.gopts)
rtest.Assert(t, err != nil, "expected error for non-existing file")
rtest.Assert(t, errors.Is(err, ErrNoSourceData), "expected ErrNoSourceData; got %v", err)
opts := BackupOptions{}
testRunBackup(t, "", dirs, opts, env.gopts)
return nil
})
}
func TestBackupSelfHealing(t *testing.T) {
@@ -444,13 +438,13 @@ func TestIncrementalBackup(t *testing.T) {
testRunBackup(t, "", []string{datadir}, opts, env.gopts)
testRunCheck(t, env.gopts)
stat1 := dirStats(t, env.repo)
stat1 := dirStats(env.repo)
rtest.OK(t, appendRandomData(testfile, incrementalSecondWrite))
testRunBackup(t, "", []string{datadir}, opts, env.gopts)
testRunCheck(t, env.gopts)
stat2 := dirStats(t, env.repo)
stat2 := dirStats(env.repo)
if stat2.size-stat1.size > incrementalFirstWrite {
t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
}
@@ -460,13 +454,14 @@ func TestIncrementalBackup(t *testing.T) {
testRunBackup(t, "", []string{datadir}, opts, env.gopts)
testRunCheck(t, env.gopts)
stat3 := dirStats(t, env.repo)
stat3 := dirStats(env.repo)
if stat3.size-stat2.size > incrementalFirstWrite {
t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
}
t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
}
// nolint: staticcheck // false positive nil pointer dereference check
func TestBackupTags(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
@@ -486,7 +481,7 @@ func TestBackupTags(t *testing.T) {
"expected no tags, got %v", newest.Tags)
parent := newest
opts.Tags = data.TagLists{[]string{"NL"}}
opts.Tags = restic.TagLists{[]string{"NL"}}
testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
testRunCheck(t, env.gopts)
newest, _ = testRunSnapshots(t, env.gopts)
@@ -502,6 +497,7 @@ func TestBackupTags(t *testing.T) {
"expected parent to be %v, got %v", parent.ID, newest.Parent)
}
// nolint: staticcheck // false positive nil pointer dereference check
func TestBackupProgramVersion(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
@@ -513,7 +509,7 @@ func TestBackupProgramVersion(t *testing.T) {
if newest == nil {
t.Fatal("expected a backup, got nil")
}
resticVersion := "restic " + global.Version
resticVersion := "restic " + version
rtest.Assert(t, newest.ProgramVersion == resticVersion,
"expected %v, got %v", resticVersion, newest.ProgramVersion)
}
@@ -571,7 +567,7 @@ func TestHardLink(t *testing.T) {
restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
testRunRestore(t, env.gopts, restoredir, snapshotID.String())
diff := directoriesContentsDiff(t, env.testdata, filepath.Join(restoredir, "testdata"))
diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
rtest.Assert(t, diff == "", "directories are not equal %v", diff)
linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
@@ -707,7 +703,7 @@ func TestBackupEmptyPassword(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
env.gopts.Password = ""
env.gopts.password = ""
env.gopts.InsecureNoPassword = true
testSetupBackupData(t, env)

View File

@@ -67,13 +67,10 @@ func TestCollectTargets(t *testing.T) {
FilesFromRaw: []string{f3.Name()},
}
targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")}, t.Logf, nil)
targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")})
rtest.OK(t, err)
sort.Strings(targets)
rtest.Equals(t, expect, targets)
_, err = collectTargets(opts, []string{filepath.Join(dir, "cmdline arg"), filepath.Join(dir, "non-existing-file")}, t.Logf, nil)
rtest.Assert(t, err == ErrInvalidSourceData, "expected error when not all targets exist")
}
func TestReadFilenamesRaw(t *testing.T) {

View File

@@ -10,14 +10,13 @@ import (
"github.com/restic/restic/internal/backend/cache"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newCacheCommand(globalOptions *global.Options) *cobra.Command {
func newCacheCommand() *cobra.Command {
var opts CacheOptions
cmd := &cobra.Command{
@@ -35,7 +34,7 @@ Exit status is 1 if there was any error.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return runCache(opts, *globalOptions, args, globalOptions.Term)
return runCache(opts, globalOptions, args)
},
}
@@ -56,9 +55,7 @@ func (opts *CacheOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.NoSize, "no-size", false, "do not output the size of the cache directories")
}
func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
if len(args) > 0 {
return errors.Fatal("the cache command expects no arguments, only options - please see `restic help cache` for usage and flags")
}
@@ -86,17 +83,17 @@ func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Te
}
if len(oldDirs) == 0 {
printer.P("no old cache dirs found")
Verbosef("no old cache dirs found\n")
return nil
}
printer.P("remove %d old cache directories", len(oldDirs))
Verbosef("remove %d old cache directories\n", len(oldDirs))
for _, item := range oldDirs {
dir := filepath.Join(cachedir, item.Name())
err = os.RemoveAll(dir)
if err != nil {
printer.E("unable to remove %v: %v", dir, err)
Warnf("unable to remove %v: %v\n", dir, err)
}
}
@@ -126,7 +123,7 @@ func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Te
}
if len(dirs) == 0 {
printer.S("no cache dirs found, basedir is %v", cachedir)
Printf("no cache dirs found, basedir is %v\n", cachedir)
return nil
}
@@ -162,8 +159,8 @@ func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Te
})
}
_ = tab.Write(gopts.Term.OutputWriter())
printer.S("%d cache dirs in %s", len(dirs), cachedir)
_ = tab.Write(globalOptions.stdout)
Printf("%d cache dirs in %s\n", len(dirs), cachedir)
return nil
}

View File

@@ -7,17 +7,14 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
)
var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
func newCatCommand(globalOptions *global.Options) *cobra.Command {
func newCatCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
Short: "Print internal objects to stdout",
@@ -36,7 +33,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runCat(cmd.Context(), *globalOptions, args, globalOptions.Term)
return runCat(cmd.Context(), globalOptions, args)
},
ValidArgs: catAllowedCmds,
}
@@ -66,14 +63,12 @@ func validateCatArgs(args []string) error {
return nil
}
func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
if err := validateCatArgs(args); err != nil {
return err
}
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
@@ -85,7 +80,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
id, err = restic.ParseID(args[1])
if err != nil {
return errors.Fatalf("unable to parse ID: %v", err)
return errors.Fatalf("unable to parse ID: %v\n", err)
}
}
@@ -96,7 +91,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err
}
printer.S(string(buf))
Println(string(buf))
return nil
case "index":
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
@@ -104,12 +99,12 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err
}
printer.S(string(buf))
Println(string(buf))
return nil
case "snapshot":
sn, _, err := data.FindSnapshot(ctx, repo, repo, args[1])
sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v", err)
return errors.Fatalf("could not find snapshot: %v\n", err)
}
buf, err := json.MarshalIndent(sn, "", " ")
@@ -117,7 +112,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err
}
printer.S(string(buf))
Println(string(buf))
return nil
case "key":
key, err := repository.LoadKey(ctx, repo, id)
@@ -130,7 +125,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err
}
printer.S(string(buf))
Println(string(buf))
return nil
case "masterkey":
buf, err := json.MarshalIndent(repo.Key(), "", " ")
@@ -138,7 +133,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err
}
printer.S(string(buf))
Println(string(buf))
return nil
case "lock":
lock, err := restic.LoadLock(ctx, repo, id)
@@ -151,7 +146,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err
}
printer.S(string(buf))
Println(string(buf))
return nil
case "pack":
@@ -163,14 +158,15 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
hash := restic.Hash(buf)
if !hash.Equal(id) {
printer.E("Warning: hash of data does not match ID, want\n %v\ngot:\n %v", id.String(), hash.String())
Warnf("Warning: hash of data does not match ID, want\n %v\ngot:\n %v\n", id.String(), hash.String())
}
_, err = term.OutputRaw().Write(buf)
_, err = globalOptions.stdout.Write(buf)
return err
case "blob":
err = repo.LoadIndex(ctx, printer)
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
@@ -185,24 +181,25 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err
}
_, err = term.OutputRaw().Write(buf)
_, err = globalOptions.stdout.Write(buf)
return err
}
return errors.Fatal("blob not found")
case "tree":
sn, subfolder, err := data.FindSnapshot(ctx, repo, repo, args[1])
sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v", err)
return errors.Fatalf("could not find snapshot: %v\n", err)
}
err = repo.LoadIndex(ctx, printer)
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
@@ -211,7 +208,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
if err != nil {
return err
}
_, err = term.OutputRaw().Write(buf)
_, err = globalOptions.stdout.Write(buf)
return err
default:

View File

@@ -15,16 +15,15 @@ import (
"github.com/restic/restic/internal/backend/cache"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
)
func newCheckCommand(globalOptions *global.Options) *cobra.Command {
func newCheckCommand() *cobra.Command {
var opts CheckOptions
cmd := &cobra.Command{
Use: "check [flags]",
@@ -48,13 +47,14 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
summary, err := runCheck(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
term, cancel := setupTermstatus()
defer cancel()
summary, err := runCheck(cmd.Context(), opts, globalOptions, args, term)
if globalOptions.JSON {
if err != nil && summary.NumErrors == 0 {
summary.NumErrors = 1
}
globalOptions.Term.Print(ui.ToJSONString(summary))
term.Print(ui.ToJSONString(summary))
}
return err
},
@@ -73,7 +73,6 @@ type CheckOptions struct {
ReadDataSubset string
CheckUnused bool
WithCache bool
data.SnapshotFilter
}
func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
@@ -87,7 +86,6 @@ func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
panic(err)
}
f.BoolVar(&opts.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
}
func checkFlags(opts CheckOptions) error {
@@ -175,7 +173,7 @@ func parsePercentage(s string) (float64, error) {
// - if the user explicitly requested --no-cache, we don't use any cache
// - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
// - by default, we use a cache in a temporary directory that is deleted after the check
func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progress.Printer) (cleanup func()) {
func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress.Printer) (cleanup func()) {
cleanup = func() {}
if opts.WithCache {
// use the default cache, no setup needed
@@ -196,7 +194,7 @@ func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progres
// use a cache in a temporary directory
err := os.MkdirAll(cachedir, 0755)
if err != nil {
printer.E("unable to create cache directory %s, disabling cache: %v", cachedir, err)
Warnf("unable to create cache directory %s, disabling cache: %v\n", cachedir, err)
gopts.NoCache = true
return cleanup
}
@@ -222,12 +220,15 @@ func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progres
return cleanup
}
func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args []string, term ui.Terminal) (checkSummary, error) {
func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) (checkSummary, error) {
summary := checkSummary{MessageType: "summary"}
if len(args) != 0 {
return summary, errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
}
var printer progress.Printer
if !gopts.JSON {
printer = ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
printer = newTerminalProgressPrinter(gopts.verbosity, term)
} else {
printer = newJSONErrorPrinter(term)
}
@@ -238,20 +239,21 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
if !gopts.NoLock {
printer.P("create exclusive lock for repository\n")
}
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
if err != nil {
return summary, err
}
defer unlock()
chkr := checker.New(repo, opts.CheckUnused)
err = chkr.LoadSnapshots(ctx, &opts.SnapshotFilter, args)
err = chkr.LoadSnapshots(ctx)
if err != nil {
return summary, err
}
printer.P("load indexes\n")
hints, errs := chkr.LoadIndex(ctx, printer)
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
hints, errs := chkr.LoadIndex(ctx, bar)
if ctx.Err() != nil {
return summary, ctx.Err()
}
@@ -259,10 +261,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
errorsFound := false
for _, hint := range hints {
switch hint.(type) {
case *repository.ErrDuplicatePacks:
case *checker.ErrDuplicatePacks:
printer.S("%s", hint.Error())
summary.HintRepairIndex = true
case *repository.ErrMixedPack:
case *checker.ErrMixedPack:
printer.S("%s", hint.Error())
summary.HintPrune = true
default:
@@ -297,7 +299,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
go chkr.Packs(ctx, errChan)
for err := range errChan {
var packErr *repository.PackError
var packErr *checker.PackError
if errors.As(err, &packErr) {
if packErr.Orphaned {
orphanedPacks++
@@ -361,7 +363,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
return summary, ctx.Err()
}
// the following block only used for tests
if opts.CheckUnused {
unused, err := chkr.UnusedBlobs(ctx)
if err != nil {
@@ -373,16 +374,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
}
}
readDataFilter, err := buildPacksFilter(opts, printer, chkr.IsFiltered())
if err != nil {
return summary, err
}
if readDataFilter != nil {
doReadData := func(packs map[restic.ID]int64) {
p := printer.NewCounter("packs")
p.SetMax(uint64(len(packs)))
errChan := make(chan error)
go chkr.ReadPacks(ctx, readDataFilter, p, errChan)
go chkr.ReadPacks(ctx, packs, p, errChan)
for err := range errChan {
errorsFound = true
@@ -395,6 +392,48 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
p.Done()
}
switch {
case opts.ReadData:
printer.P("read all data\n")
doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
case opts.ReadDataSubset != "":
var packs map[restic.ID]int64
dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
if err == nil {
bucket := dataSubset[0]
totalBuckets := dataSubset[1]
packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
packCount := uint64(len(packs))
printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
percentage, err := parsePercentage(opts.ReadDataSubset)
if err == nil {
packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
printer.P("read %.1f%% of data packs\n", percentage)
}
} else {
repoSize := int64(0)
allPacks := chkr.GetPacks()
for _, size := range allPacks {
repoSize += size
}
if repoSize == 0 {
return summary, errors.Fatal("Cannot read from a repository having size 0")
}
subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
if subsetSize > repoSize {
subsetSize = repoSize
}
packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
percentage := float64(subsetSize) / float64(repoSize) * 100.0
printer.P("read %d bytes (%.1f%%) of data packs\n", subsetSize, percentage)
}
if packs == nil {
return summary, errors.Fatal("internal error: failed to select packs to check")
}
doReadData(packs)
}
if len(salvagePacks) > 0 {
printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
for id := range salvagePacks {
@@ -418,64 +457,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
return summary, nil
}
func buildPacksFilter(opts CheckOptions, printer progress.Printer,
filteredStatus bool) (func(packs map[restic.ID]int64) map[restic.ID]int64, error) {
typeData := ""
if filteredStatus {
typeData = "filtered "
}
switch {
case opts.ReadData:
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
printer.P("read all %sdata", typeData)
return packs
}, nil
case opts.ReadDataSubset != "":
dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
if err == nil {
bucket := dataSubset[0]
totalBuckets := dataSubset[1]
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
packCount := uint64(len(packs))
packs = selectPacksByBucket(packs, bucket, totalBuckets)
printer.P("read group #%d of %d %sdata packs (out of total %d packs in %d groups", bucket, len(packs), typeData, packCount, totalBuckets)
return packs
}, nil
} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
percentage, err := parsePercentage(opts.ReadDataSubset)
if err != nil {
return nil, err
}
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
printer.P("read %.1f%% of %spackfiles", percentage, typeData)
return selectRandomPacksByPercentage(packs, percentage)
}, nil
}
repoSize := int64(0)
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
for _, size := range packs {
repoSize += size
}
subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
if subsetSize > repoSize {
subsetSize = repoSize
}
if repoSize > 0 {
packs = selectRandomPacksByFileSize(packs, subsetSize, repoSize)
}
percentage := float64(subsetSize) / float64(repoSize) * 100.0
if repoSize == 0 {
percentage = 100
}
printer.P("read %d bytes (%.1f%%) of %sdata packs\n", subsetSize, percentage, typeData)
return packs
}, nil
}
return nil, nil
}
// selectPacksByBucket selects subsets of packs by ranges of buckets.
func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 {
packs := make(map[restic.ID]int64)
@@ -547,10 +528,6 @@ func (*jsonErrorPrinter) NewCounter(_ string) *progress.Counter {
return nil
}
func (*jsonErrorPrinter) NewCounterTerminalOnly(_ string) *progress.Counter {
return nil
}
func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
status := checkError{
MessageType: "error",
@@ -560,6 +537,5 @@ func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
}
func (*jsonErrorPrinter) S(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) P(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) PT(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) V(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {}

View File

@@ -1,101 +1,39 @@
package main
import (
"bytes"
"context"
"strings"
"testing"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunCheck(t testing.TB, gopts global.Options) {
func testRunCheck(t testing.TB, gopts GlobalOptions) {
t.Helper()
output, err := testRunCheckOutput(t, gopts, true)
output, err := testRunCheckOutput(gopts, true)
if err != nil {
t.Error(output)
t.Fatalf("unexpected error: %+v", err)
}
}
func testRunCheckMustFail(t testing.TB, gopts global.Options) {
func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
t.Helper()
_, err := testRunCheckOutput(t, gopts, false)
_, err := testRunCheckOutput(gopts, false)
rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository")
}
func testRunCheckOutput(t testing.TB, gopts global.Options, checkUnused bool) (string, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
buf := bytes.NewBuffer(nil)
gopts.stdout = buf
err := withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
opts := CheckOptions{
ReadData: true,
CheckUnused: checkUnused,
}
_, err := runCheck(context.TODO(), opts, gopts, nil, gopts.Term)
_, err := runCheck(context.TODO(), opts, gopts, nil, term)
return err
})
return buf.String(), err
}
func testRunCheckOutputWithOpts(t testing.TB, gopts global.Options, opts CheckOptions, args []string) (string, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
gopts.Verbosity = 2
_, err := runCheck(context.TODO(), opts, gopts, args, gopts.Term)
return err
})
return buf.String(), err
}
func TestCheckWithSnaphotFilter(t *testing.T) {
testCases := []struct {
opts CheckOptions
args []string
expectedOutput string
}{
{ // full --read-data, all snapshots
CheckOptions{ReadData: true},
nil,
"4 / 4 packs",
},
{ // full --read-data, all snapshots
CheckOptions{ReadData: true},
nil,
"2 / 2 snapshots",
},
{ // full --read-data, latest snapshot
CheckOptions{ReadData: true},
[]string{"latest"},
"2 / 2 packs",
},
{ // full --read-data, latest snapshot
CheckOptions{ReadData: true},
[]string{"latest"},
"1 / 1 snapshots",
},
{ // --read-data-subset, latest snapshot
CheckOptions{ReadDataSubset: "1%"},
[]string{"latest"},
"1 / 1 packs",
},
{ // --read-data-subset, latest snapshot
CheckOptions{ReadDataSubset: "1%"},
[]string{"latest"},
"filtered",
},
}
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
opts := BackupOptions{}
testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts)
testRunBackup(t, env.testdata+"/0", []string{"0/9"}, opts, env.gopts)
for _, testCase := range testCases {
output, err := testRunCheckOutputWithOpts(t, env.gopts, testCase.opts, testCase.args)
rtest.OK(t, err)
hasOutput := strings.Contains(output, testCase.expectedOutput)
rtest.Assert(t, hasOutput, `expected to find substring %q, but did not find it`, testCase.expectedOutput)
}
}

View File

@@ -9,7 +9,6 @@ import (
"testing"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/progress"
@@ -203,7 +202,7 @@ func TestPrepareCheckCache(t *testing.T) {
err := os.Remove(tmpDirBase)
rtest.OK(t, err)
}
gopts := global.Options{CacheDir: tmpDirBase}
gopts := GlobalOptions{CacheDir: tmpDirBase}
cleanup := prepareCheckCache(testCase.opts, &gopts, &progress.NoopPrinter{})
files, err := os.ReadDir(tmpDirBase)
rtest.OK(t, err)
@@ -233,7 +232,7 @@ func TestPrepareCheckCache(t *testing.T) {
}
func TestPrepareDefaultCheckCache(t *testing.T) {
gopts := global.Options{CacheDir: ""}
gopts := GlobalOptions{CacheDir: ""}
cleanup := prepareCheckCache(CheckOptions{}, &gopts, &progress.NoopPrinter{})
_, err := os.ReadDir(gopts.CacheDir)
rtest.OK(t, err)

View File

@@ -3,24 +3,18 @@ package main
import (
"context"
"fmt"
"iter"
"sync"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newCopyCommand(globalOptions *global.Options) *cobra.Command {
func newCopyCommand() *cobra.Command {
var opts CopyOptions
cmd := &cobra.Command{
Use: "copy [flags] [snapshotID ...]",
@@ -52,8 +46,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runCopy(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
return runCopy(cmd.Context(), opts, globalOptions, args)
},
}
@@ -63,51 +56,17 @@ Exit status is 12 if the password is incorrect.
// CopyOptions bundles all options for the copy command.
type CopyOptions struct {
global.SecondaryRepoOptions
data.SnapshotFilter
secondaryRepoOptions
restic.SnapshotFilter
}
func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) {
opts.SecondaryRepoOptions.AddFlags(f, "destination", "to copy snapshots from")
opts.secondaryRepoOptions.AddFlags(f, "destination", "to copy snapshots from")
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
}
// collectAllSnapshots: select all snapshot trees to be copied
func collectAllSnapshots(ctx context.Context, opts CopyOptions,
srcSnapshotLister restic.Lister, srcRepo restic.Repository,
dstSnapshotByOriginal map[restic.ID][]*data.Snapshot, args []string, printer progress.Printer,
) iter.Seq[*data.Snapshot] {
return func(yield func(*data.Snapshot) bool) {
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args, printer) {
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
if sn.Original != nil {
srcOriginal = *sn.Original
}
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
printer.V("\n%v", sn)
printer.V("skipping source snapshot %s, was already copied to snapshot %s", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
}
if isCopy {
continue
}
}
if !yield(sn) {
return
}
}
}
}
func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
secondaryGopts, isFromRepo, err := opts.SecondaryRepoOptions.FillGlobalOpts(ctx, gopts, "destination")
func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
if err != nil {
return err
}
@@ -116,13 +75,13 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [
gopts, secondaryGopts = secondaryGopts, gopts
}
ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
defer unlock()
ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false, printer)
ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false)
if err != nil {
return err
}
@@ -139,16 +98,18 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [
}
debug.Log("Loading source index")
if err := srcRepo.LoadIndex(ctx, printer); err != nil {
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err := srcRepo.LoadIndex(ctx, bar); err != nil {
return err
}
bar = newIndexProgress(gopts.Quiet, gopts.JSON)
debug.Log("Loading destination index")
if err := dstRepo.LoadIndex(ctx, printer); err != nil {
if err := dstRepo.LoadIndex(ctx, bar); err != nil {
return err
}
dstSnapshotByOriginal := make(map[restic.ID][]*data.Snapshot)
for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil, printer) {
dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) {
if sn.Original != nil && !sn.Original.IsNull() {
dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
}
@@ -159,16 +120,53 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [
return ctx.Err()
}
selectedSnapshots := collectAllSnapshots(ctx, opts, srcSnapshotLister, srcRepo, dstSnapshotByOriginal, args, printer)
// remember already processed trees across all snapshots
visitedTrees := restic.NewIDSet()
if err := copyTreeBatched(ctx, srcRepo, dstRepo, selectedSnapshots, printer); err != nil {
return err
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) {
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
if sn.Original != nil {
srcOriginal = *sn.Original
}
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
Verboseff("\n%v\n", sn)
Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
}
if isCopy {
continue
}
}
Verbosef("\n%v\n", sn)
Verbosef(" copy started, this may take a while...\n")
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
return err
}
debug.Log("tree copied")
// save snapshot
sn.Parent = nil // Parent does not have relevance in the new repo.
// Use Original as a persistent snapshot ID
if sn.Original == nil {
sn.Original = sn.ID()
}
newID, err := restic.SaveSnapshot(ctx, dstRepo, sn)
if err != nil {
return err
}
Verbosef("snapshot %s saved\n", newID.Str())
}
return ctx.Err()
}
func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool {
func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
// everything except Parent and Original must match
if !sna.Time.Equal(snb.Time) || !sna.Tree.Equal(*snb.Tree) || sna.Hostname != snb.Hostname ||
sna.Username != snb.Username || sna.UID != snb.UID || sna.GID != snb.GID ||
@@ -187,158 +185,72 @@ func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool {
return true
}
// copyTreeBatched copies multiple snapshots in one go. Snapshots are written after
// data equivalent to at least 10 packfiles was written.
func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error {
// remember already processed trees across all snapshots
visitedTrees := srcRepo.NewAssociatedBlobSet()
targetSize := uint64(dstRepo.PackSize()) * 100
minDuration := 1 * time.Minute
// use pull-based iterator to allow iteration in multiple steps
next, stop := iter.Pull(selectedSnapshots)
defer stop()
for {
var batch []*data.Snapshot
batchSize := uint64(0)
startTime := time.Now()
// call WithBlobUploader() once and then loop over all selectedSnapshots
err := dstRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for batchSize < targetSize || time.Since(startTime) < minDuration {
sn, ok := next()
if !ok {
break
}
batch = append(batch, sn)
printer.P("\n%v", sn)
printer.P(" copy started, this may take a while...")
sizeBlobs, err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, printer, uploader)
if err != nil {
return err
}
debug.Log("tree copied")
batchSize += sizeBlobs
}
return nil
})
if err != nil {
return err
}
// if no snapshots were processed in this batch, we're done
if len(batch) == 0 {
break
}
// add a newline to separate saved snapshot messages from the other messages
if len(batch) > 1 {
printer.P("")
}
// save all the snapshots
for _, sn := range batch {
err := copySaveSnapshot(ctx, sn, dstRepo, printer)
if err != nil {
return err
}
}
}
return nil
}
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) {
visitedTrees restic.IDSet, rootTreeID restic.ID, quiet bool) error {
copyBlobs := srcRepo.NewAssociatedBlobSet()
wg, wgCtx := errgroup.WithContext(ctx)
treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
visited := visitedTrees.Has(treeID)
visitedTrees.Insert(treeID)
return visited
}, nil)
copyBlobs := restic.NewBlobSet()
packList := restic.NewIDSet()
var lock sync.Mutex
enqueue := func(h restic.BlobHandle) {
lock.Lock()
defer lock.Unlock()
if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
pb := srcRepo.LookupBlob(h.Type, h.ID)
copyBlobs.Insert(h)
for _, p := range pb {
packList.Insert(p.PackID)
}
pb := srcRepo.LookupBlob(h.Type, h.ID)
copyBlobs.Insert(h)
for _, p := range pb {
packList.Insert(p.PackID)
}
}
err := data.StreamTrees(ctx, srcRepo, restic.IDs{rootTreeID}, nil, func(treeID restic.ID) bool {
handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob}
visited := visitedTrees.Has(handle)
visitedTrees.Insert(handle)
return visited
}, func(treeID restic.ID, err error, nodes data.TreeNodeIterator) error {
if err != nil {
return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
}
// copy raw tree bytes to avoid problems if the serialization changes
enqueue(restic.BlobHandle{ID: treeID, Type: restic.TreeBlob})
for item := range nodes {
if item.Error != nil {
return item.Error
wg.Go(func() error {
for tree := range treeStream {
if tree.Error != nil {
return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
}
// Recursion into directories is handled by StreamTrees
// Copy the blobs for this file.
for _, blobID := range item.Node.Content {
enqueue(restic.BlobHandle{Type: restic.DataBlob, ID: blobID})
// Do we already have this tree blob?
treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok {
// copy raw tree bytes to avoid problems if the serialization changes
enqueue(treeHandle)
}
for _, entry := range tree.Nodes {
// Recursion into directories is handled by StreamTrees
// Copy the blobs for this file.
for _, blobID := range entry.Content {
h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
enqueue(h)
}
}
}
}
return nil
})
if err != nil {
return 0, err
}
sizeBlobs := copyStats(srcRepo, copyBlobs, packList, printer)
bar := printer.NewCounter("packs copied")
err = repository.CopyBlobs(ctx, srcRepo, dstRepo, uploader, packList, copyBlobs, bar, printer.P)
if err != nil {
return 0, errors.Fatalf("%s", err)
}
return sizeBlobs, nil
}
// copyStats: print statistics for the blobs to be copied
func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, packList restic.IDSet, printer progress.Printer) uint64 {
// count and size
countBlobs := 0
sizeBlobs := uint64(0)
for blob := range copyBlobs.Keys() {
for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) {
countBlobs++
sizeBlobs += uint64(blob.Length)
break
}
}
printer.V(" copy %d blobs with disk size %s in %d packfiles\n",
countBlobs, ui.FormatBytes(uint64(sizeBlobs)), len(packList))
return sizeBlobs
}
func copySaveSnapshot(ctx context.Context, sn *data.Snapshot, dstRepo restic.Repository, printer progress.Printer) error {
sn.Parent = nil // Parent does not have relevance in the new repo.
// Use Original as a persistent snapshot ID
if sn.Original == nil {
sn.Original = sn.ID()
}
newID, err := data.SaveSnapshot(ctx, dstRepo, sn)
err := wg.Wait()
if err != nil {
return err
}
printer.P("snapshot %s saved, copied from source snapshot %s", newID.Str(), sn.ID().Str())
bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
_, err = repository.Repack(
ctx,
srcRepo,
dstRepo,
packList,
copyBlobs,
bar,
func(msg string, args ...interface{}) { fmt.Printf(msg+"\n", args...) },
)
bar.Done()
if err != nil {
return errors.Fatal(err.Error())
}
return nil
}

View File

@@ -6,28 +6,23 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
)
func testRunCopy(t testing.TB, srcGopts global.Options, dstGopts global.Options) {
func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
gopts := srcGopts
gopts.Repo = dstGopts.Repo
gopts.Password = dstGopts.Password
gopts.password = dstGopts.password
gopts.InsecureNoPassword = dstGopts.InsecureNoPassword
copyOpts := CopyOptions{
SecondaryRepoOptions: global.SecondaryRepoOptions{
secondaryRepoOptions: secondaryRepoOptions{
Repo: srcGopts.Repo,
Password: srcGopts.Password,
password: srcGopts.password,
InsecureNoPassword: srcGopts.InsecureNoPassword,
},
}
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runCopy(context.TODO(), copyOpts, gopts, nil, gopts.Term)
}))
rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
}
func TestCopy(t *testing.T) {
@@ -50,8 +45,8 @@ func TestCopy(t *testing.T) {
copiedSnapshotIDs := testListSnapshots(t, env2.gopts, 3)
// Check that the copies size seems reasonable
stat := dirStats(t, env.repo)
stat2 := dirStats(t, env2.repo)
stat := dirStats(env.repo)
stat2 := dirStats(env2.repo)
sizeDiff := int64(stat.size) - int64(stat2.size)
if sizeDiff < 0 {
sizeDiff = -sizeDiff
@@ -74,7 +69,7 @@ func TestCopy(t *testing.T) {
testRunRestore(t, env2.gopts, restoredir, snapshotID.String())
foundMatch := false
for cmpdir := range origRestores {
diff := directoriesContentsDiff(t, restoredir, cmpdir)
diff := directoriesContentsDiff(restoredir, cmpdir)
if diff == "" {
delete(origRestores, cmpdir)
foundMatch = true
@@ -85,41 +80,6 @@ func TestCopy(t *testing.T) {
}
rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots")
// check that snapshots were properly batched while copying
_, _, countBlobs := testPackAndBlobCounts(t, env.gopts)
countTreePacksDst, countDataPacksDst, countBlobsDst := testPackAndBlobCounts(t, env2.gopts)
rtest.Equals(t, countBlobs, countBlobsDst, "expected blob count in boths repos to be equal")
rtest.Equals(t, countTreePacksDst, 1, "expected 1 tree packfile")
rtest.Equals(t, countDataPacksDst, 1, "expected 1 data packfile")
}
func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks int, countDataPacks int, countBlobs int) {
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
rtest.OK(t, err)
defer unlock()
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(context.TODO(), id, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
switch blobs[0].Type {
case restic.TreeBlob:
countTreePacks++
case restic.DataBlob:
countDataPacks++
}
countBlobs += len(blobs)
return nil
}))
return nil
}))
return countTreePacks, countDataPacks, countBlobs
}
func TestCopyIncremental(t *testing.T) {
@@ -182,7 +142,7 @@ func TestCopyToEmptyPassword(t *testing.T) {
defer cleanup()
env2, cleanup2 := withTestEnvironment(t)
defer cleanup2()
env2.gopts.Password = ""
env2.gopts.password = ""
env2.gopts.InsecureNoPassword = true
testSetupBackupData(t, env)

View File

@@ -1,4 +1,5 @@
//go:build debug
// +build debug
package main
@@ -21,36 +22,32 @@ import (
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
)
func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) {
func registerDebugCommand(cmd *cobra.Command) {
cmd.AddCommand(
newDebugCommand(globalOptions),
newDebugCommand(),
)
}
func newDebugCommand(globalOptions *global.Options) *cobra.Command {
func newDebugCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "debug",
Short: "Debug commands",
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
}
cmd.AddCommand(newDebugDumpCommand(globalOptions))
cmd.AddCommand(newDebugExamineCommand(globalOptions))
cmd.AddCommand(newDebugDumpCommand())
cmd.AddCommand(newDebugExamineCommand())
return cmd
}
func newDebugDumpCommand(globalOptions *global.Options) *cobra.Command {
func newDebugDumpCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dump [indexes|snapshots|all|packs]",
Short: "Dump data structures",
@@ -69,13 +66,13 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDebugDump(cmd.Context(), *globalOptions, args, globalOptions.Term)
return runDebugDump(cmd.Context(), globalOptions, args)
},
}
return cmd
}
func newDebugExamineCommand(globalOptions *global.Options) *cobra.Command {
func newDebugExamineCommand() *cobra.Command {
var opts DebugExamineOptions
cmd := &cobra.Command{
@@ -83,7 +80,7 @@ func newDebugExamineCommand(globalOptions *global.Options) *cobra.Command {
Short: "Examine a pack file",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDebugExamine(cmd.Context(), *globalOptions, opts, args, globalOptions.Term)
return runDebugExamine(cmd.Context(), globalOptions, opts, args)
},
}
@@ -116,7 +113,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
}
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
return data.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *data.Snapshot, err error) error {
return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
if err != nil {
return err
}
@@ -144,13 +141,13 @@ type Blob struct {
Offset uint `json:"offset"`
}
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error {
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
printer.E("error for pack %v: %v", id.Str(), err)
Warnf("error for pack %v: %v\n", id.Str(), err)
return nil
}
@@ -173,9 +170,9 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer,
})
}
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error {
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id)
Printf("index_id: %v\n", id)
if err != nil {
return err
}
@@ -184,14 +181,12 @@ func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Wr
})
}
func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified")
}
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
@@ -201,20 +196,20 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
switch tpe {
case "indexes":
return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
return dumpIndexes(ctx, repo, globalOptions.stdout)
case "snapshots":
return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
return debugPrintSnapshots(ctx, repo, globalOptions.stdout)
case "packs":
return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer)
return printPacks(ctx, repo, globalOptions.stdout)
case "all":
printer.S("snapshots:")
err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter())
Printf("snapshots:\n")
err := debugPrintSnapshots(ctx, repo, globalOptions.stdout)
if err != nil {
return err
}
printer.S("indexes:")
err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer)
Printf("\nindexes:\n")
err = dumpIndexes(ctx, repo, globalOptions.stdout)
if err != nil {
return err
}
@@ -225,11 +220,11 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
}
}
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte {
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool) []byte {
if bytewise {
printer.S(" trying to repair blob by finding a broken byte")
Printf(" trying to repair blob by finding a broken byte\n")
} else {
printer.S(" trying to repair blob with single bit flip")
Printf(" trying to repair blob with single bit flip\n")
}
ch := make(chan int)
@@ -239,7 +234,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
var found bool
workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0))
Printf(" spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ {
wg.Go(func() error {
// make a local copy of the buffer
@@ -253,9 +248,9 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil {
printer.S("")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext))
Printf("\n")
Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
Printf(" hash is %v\n", restic.Hash(plaintext))
close(done)
found = true
fixed = plaintext
@@ -296,7 +291,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
select {
case ch <- i:
case <-done:
printer.S(" done after %v", time.Since(start))
Printf(" done after %v\n", time.Since(start))
return nil
}
@@ -306,7 +301,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now()
}
@@ -319,7 +314,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
}
if !found {
printer.S("\n blob could not be repaired")
Printf("\n blob could not be repaired\n")
}
return fixed
}
@@ -340,7 +335,7 @@ func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
return out
}
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob, printer progress.Printer) error {
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
@@ -352,11 +347,17 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
return err
}
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
wg, ctx := errgroup.WithContext(ctx)
if opts.ReuploadBlobs {
repo.StartPackUploader(ctx, wg)
}
wg.Go(func() error {
for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length)
Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(pack) {
printer.E("skipping truncated blob")
Warnf("skipping truncated blob\n")
continue
}
buf := pack[blob.Offset : blob.Offset+blob.Length]
@@ -367,16 +368,16 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
outputPrefix := ""
filePrefix := ""
if err != nil {
printer.E("error decrypting blob: %v", err)
Warnf("error decrypting blob: %v\n", err)
if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer)
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte)
}
if plaintext != nil {
outputPrefix = "repaired "
filePrefix = "repaired-"
} else {
plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer)
err = storePlainBlob(blob.ID, "damaged-", plaintext)
if err != nil {
return err
}
@@ -387,7 +388,7 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil {
printer.S(" failed to decompress blob %v", blob.ID)
Printf(" failed to decompress blob %v\n", blob.ID)
}
if decompressed != nil {
plaintext = decompressed
@@ -397,32 +398,37 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
id := restic.Hash(plaintext)
var prefix string
if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID)
Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-"
} else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id)
Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
prefix = "correct-"
}
if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer)
err = storePlainBlob(id, filePrefix+prefix, plaintext)
if err != nil {
return err
}
}
if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true)
_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil {
return err
}
printer.S(" uploaded %v %v", blob.Type, id)
Printf(" uploaded %v %v\n", blob.Type, id)
}
}
if opts.ReuploadBlobs {
return repo.Flush(ctx)
}
return nil
})
return err
return wg.Wait()
}
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error {
func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename)
if err != nil {
@@ -440,18 +446,16 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.
return err
}
printer.S("decrypt of blob %v stored at %v", id, filename)
Printf("decrypt of blob %v stored at %v\n", id, filename)
return nil
}
func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
if opts.ExtractPack && gopts.NoLock {
return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive")
}
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
@@ -463,7 +467,7 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
if err != nil {
id, err = restic.Find(ctx, repo, restic.PackFile, name)
if err != nil {
printer.E("error: %v", err)
Warnf("error: %v\n", err)
continue
}
}
@@ -474,15 +478,16 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
return errors.Fatal("no pack files to examine")
}
err = repo.LoadIndex(ctx, printer)
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
for _, id := range ids {
err := examinePack(ctx, opts, repo, id, printer)
err := examinePack(ctx, opts, repo, id)
if err != nil {
printer.E("error: %v", err)
Warnf("error: %v\n", err)
}
if err == context.Canceled {
break
@@ -491,24 +496,24 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
return nil
}
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error {
printer.S("examine %v", id)
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error {
Printf("examine %v\n", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
printer.S(" file size is %v", len(buf))
Printf(" file size is %v\n", len(buf))
gotID := restic.Hash(buf)
if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID)
Printf(" wanted hash %v, got %v\n", id, gotID)
} else {
printer.S(" hash for file content matches")
Printf(" hash for file content matches\n")
}
printer.S(" ========================================")
printer.S(" looking for info in the indexes")
Printf(" ========================================\n")
Printf(" looking for info in the indexes\n")
blobsLoaded := false
// examine all data the indexes have for the pack file
@@ -518,32 +523,32 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo
continue
}
checkPackSize(blobs, len(buf), printer)
checkPackSize(blobs, len(buf))
err = loadBlobs(ctx, opts, repo, id, blobs, printer)
err = loadBlobs(ctx, opts, repo, id, blobs)
if err != nil {
printer.E("error: %v", err)
Warnf("error: %v\n", err)
} else {
blobsLoaded = true
}
}
printer.S(" ========================================")
printer.S(" inspect the pack itself")
Printf(" ========================================\n")
Printf(" inspect the pack itself\n")
blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
checkPackSize(blobs, len(buf), printer)
checkPackSize(blobs, len(buf))
if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer)
return loadBlobs(ctx, opts, repo, id, blobs)
}
return nil
}
func checkPackSize(blobs []restic.Blob, fileSize int, printer progress.Printer) {
func checkPackSize(blobs []restic.Blob, fileSize int) {
// track current size and offset
var size, offset uint64
@@ -552,9 +557,9 @@ func checkPackSize(blobs []restic.Blob, fileSize int, printer progress.Printer)
})
for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length)
Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset)
Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset)
}
offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length)
@@ -562,8 +567,8 @@ func checkPackSize(blobs []restic.Blob, fileSize int, printer progress.Printer)
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize)
Printf(" file sizes do not match: computed %v, file size is %v\n", size, fileSize)
} else {
printer.S(" file sizes match")
Printf(" file sizes match\n")
}
}

View File

@@ -2,11 +2,8 @@
package main
import (
"github.com/restic/restic/internal/global"
"github.com/spf13/cobra"
)
import "github.com/spf13/cobra"
func registerDebugCommand(_ *cobra.Command, _ *global.Options) {
func registerDebugCommand(_ *cobra.Command) {
// No commands to register in non-debug mode
}

View File

@@ -5,18 +5,17 @@ import (
"encoding/json"
"path"
"reflect"
"sort"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newDiffCommand(globalOptions *global.Options) *cobra.Command {
func newDiffCommand() *cobra.Command {
var opts DiffOptions
cmd := &cobra.Command{
@@ -53,7 +52,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDiff(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
return runDiff(cmd.Context(), opts, globalOptions, args)
},
}
@@ -70,10 +69,10 @@ func (opts *DiffOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.ShowMetadata, "metadata", false, "print changes in metadata")
}
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*data.Snapshot, string, error) {
sn, subfolder, err := data.FindSnapshot(ctx, be, repo, desc)
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil {
return nil, "", errors.Fatalf("%s", err)
return nil, "", errors.Fatal(err.Error())
}
return sn, subfolder, err
}
@@ -83,7 +82,6 @@ type Comparer struct {
repo restic.BlobLoader
opts DiffOptions
printChange func(change *Change)
printError func(string, ...interface{})
}
type Change struct {
@@ -107,15 +105,15 @@ type DiffStat struct {
}
// Add adds stats information for node to s.
func (s *DiffStat) Add(node *data.Node) {
func (s *DiffStat) Add(node *restic.Node) {
if node == nil {
return
}
switch node.Type {
case data.NodeTypeFile:
case restic.NodeTypeFile:
s.Files++
case data.NodeTypeDir:
case restic.NodeTypeDir:
s.Dirs++
default:
s.Others++
@@ -123,13 +121,13 @@ func (s *DiffStat) Add(node *data.Node) {
}
// addBlobs adds the blobs of node to s.
func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
func addBlobs(bs restic.BlobSet, node *restic.Node) {
if node == nil {
return
}
switch node.Type {
case data.NodeTypeFile:
case restic.NodeTypeFile:
for _, blob := range node.Content {
h := restic.BlobHandle{
ID: blob,
@@ -137,7 +135,7 @@ func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
}
bs.Insert(h)
}
case data.NodeTypeDir:
case restic.NodeTypeDir:
h := restic.BlobHandle{
ID: *node.Subtree,
Type: restic.TreeBlob,
@@ -147,18 +145,18 @@ func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
}
type DiffStatsContainer struct {
MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.AssociatedBlobSet `json:"-"`
MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
}
// updateBlobs updates the blob counters in the stats struct.
func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *DiffStat, printError func(string, ...interface{})) {
for h := range blobs.Keys() {
func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
for h := range blobs {
switch h.Type {
case restic.DataBlob:
stats.DataBlobs++
@@ -168,7 +166,7 @@ func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *Diff
size, found := repo.LookupBlobSize(h.Type, h.ID)
if !found {
printError("unable to find blob size for %v", h)
Warnf("unable to find blob size for %v\n", h)
continue
}
@@ -176,33 +174,30 @@ func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *Diff
}
}
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.AssociatedBlobSet, prefix string, id restic.ID) error {
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
debug.Log("print %v tree %v", mode, id)
tree, err := data.LoadTree(ctx, c.repo, id)
tree, err := restic.LoadTree(ctx, c.repo, id)
if err != nil {
return err
}
for item := range tree {
if item.Error != nil {
return item.Error
}
for _, node := range tree.Nodes {
if ctx.Err() != nil {
return ctx.Err()
}
node := item.Node
name := path.Join(prefix, node.Name)
if node.Type == data.NodeTypeDir {
if node.Type == restic.NodeTypeDir {
name += "/"
}
c.printChange(NewChange(name, mode))
stats.Add(node)
addBlobs(blobs, node)
if node.Type == data.NodeTypeDir {
if node.Type == restic.NodeTypeDir {
err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
if err != nil && err != context.Canceled {
c.printError("error: %v", err)
Warnf("error: %v\n", err)
}
}
}
@@ -210,28 +205,24 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
return ctx.Err()
}
func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSet, id restic.ID) error {
func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
debug.Log("print tree %v", id)
tree, err := data.LoadTree(ctx, c.repo, id)
tree, err := restic.LoadTree(ctx, c.repo, id)
if err != nil {
return err
}
for item := range tree {
if item.Error != nil {
return item.Error
}
for _, node := range tree.Nodes {
if ctx.Err() != nil {
return ctx.Err()
}
node := item.Node
addBlobs(blobs, node)
if node.Type == data.NodeTypeDir {
if node.Type == restic.NodeTypeDir {
err := c.collectDir(ctx, blobs, *node.Subtree)
if err != nil && err != context.Canceled {
c.printError("error: %v", err)
Warnf("error: %v\n", err)
}
}
}
@@ -239,41 +230,56 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSe
return ctx.Err()
}
func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) {
names := make(map[string]struct{})
tree1Nodes = make(map[string]*restic.Node)
for _, node := range tree1.Nodes {
tree1Nodes[node.Name] = node
names[node.Name] = struct{}{}
}
tree2Nodes = make(map[string]*restic.Node)
for _, node := range tree2.Nodes {
tree2Nodes[node.Name] = node
names[node.Name] = struct{}{}
}
uniqueNames = make([]string, 0, len(names))
for name := range names {
uniqueNames = append(uniqueNames, name)
}
sort.Strings(uniqueNames)
return tree1Nodes, tree2Nodes, uniqueNames
}
func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
debug.Log("diffing %v to %v", id1, id2)
tree1, err := data.LoadTree(ctx, c.repo, id1)
tree1, err := restic.LoadTree(ctx, c.repo, id1)
if err != nil {
return err
}
tree2, err := data.LoadTree(ctx, c.repo, id2)
tree2, err := restic.LoadTree(ctx, c.repo, id2)
if err != nil {
return err
}
for dt := range data.DualTreeIterator(tree1, tree2) {
if dt.Error != nil {
return dt.Error
}
tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2)
for _, name := range names {
if ctx.Err() != nil {
return ctx.Err()
}
node1 := dt.Tree1
node2 := dt.Tree2
var name string
if node1 != nil {
name = node1.Name
} else {
name = node2.Name
}
node1, t1 := tree1Nodes[name]
node2, t2 := tree2Nodes[name]
addBlobs(stats.BlobsBefore, node1)
addBlobs(stats.BlobsAfter, node2)
switch {
case node1 != nil && node2 != nil:
case t1 && t2:
name := path.Join(prefix, name)
mod := ""
@@ -281,12 +287,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
mod += "T"
}
if node2.Type == data.NodeTypeDir {
if node2.Type == restic.NodeTypeDir {
name += "/"
}
if node1.Type == data.NodeTypeFile &&
node2.Type == data.NodeTypeFile &&
if node1.Type == restic.NodeTypeFile &&
node2.Type == restic.NodeTypeFile &&
!reflect.DeepEqual(node1.Content, node2.Content) {
mod += "M"
stats.ChangedFiles++
@@ -308,7 +314,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
c.printChange(NewChange(name, mod))
}
if node1.Type == data.NodeTypeDir && node2.Type == data.NodeTypeDir {
if node1.Type == restic.NodeTypeDir && node2.Type == restic.NodeTypeDir {
var err error
if (*node1.Subtree).Equal(*node2.Subtree) {
err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree)
@@ -316,35 +322,35 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
err = c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree)
}
if err != nil && err != context.Canceled {
c.printError("error: %v", err)
Warnf("error: %v\n", err)
}
}
case node1 != nil && node2 == nil:
case t1 && !t2:
prefix := path.Join(prefix, name)
if node1.Type == data.NodeTypeDir {
if node1.Type == restic.NodeTypeDir {
prefix += "/"
}
c.printChange(NewChange(prefix, "-"))
stats.Removed.Add(node1)
if node1.Type == data.NodeTypeDir {
if node1.Type == restic.NodeTypeDir {
err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
if err != nil && err != context.Canceled {
c.printError("error: %v", err)
Warnf("error: %v\n", err)
}
}
case node1 == nil && node2 != nil:
case !t1 && t2:
prefix := path.Join(prefix, name)
if node2.Type == data.NodeTypeDir {
if node2.Type == restic.NodeTypeDir {
prefix += "/"
}
c.printChange(NewChange(prefix, "+"))
stats.Added.Add(node2)
if node2.Type == data.NodeTypeDir {
if node2.Type == restic.NodeTypeDir {
err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
if err != nil && err != context.Canceled {
c.printError("error: %v", err)
Warnf("error: %v\n", err)
}
}
}
@@ -353,14 +359,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
return ctx.Err()
}
func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args []string, term ui.Terminal) error {
func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatalf("specify two snapshot IDs")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
@@ -382,9 +386,10 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
}
if !gopts.JSON {
printer.P("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
}
if err = repo.LoadIndex(ctx, printer); err != nil {
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}
@@ -396,31 +401,30 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
}
sn1.Tree, err = data.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
if err != nil {
return err
}
sn2.Tree, err = data.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
if err != nil {
return err
}
c := &Comparer{
repo: repo,
opts: opts,
printError: printer.E,
repo: repo,
opts: opts,
printChange: func(change *Change) {
printer.S("%-5s%v", change.Modifier, change.Path)
Printf("%-5s%v\n", change.Modifier, change.Path)
},
}
if gopts.JSON {
enc := json.NewEncoder(gopts.Term.OutputWriter())
enc := json.NewEncoder(globalOptions.stdout)
c.printChange = func(change *Change) {
err := enc.Encode(change)
if err != nil {
printer.E("JSON encode failed: %v", err)
Warnf("JSON encode failed: %v\n", err)
}
}
}
@@ -433,9 +437,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
MessageType: "statistics",
SourceSnapshot: args[0],
TargetSnapshot: args[1],
BlobsBefore: repo.NewAssociatedBlobSet(),
BlobsAfter: repo.NewAssociatedBlobSet(),
BlobsCommon: repo.NewAssociatedBlobSet(),
BlobsBefore: restic.NewBlobSet(),
BlobsAfter: restic.NewBlobSet(),
BlobsCommon: restic.NewBlobSet(),
}
stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
@@ -446,23 +450,23 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
}
both := stats.BlobsBefore.Intersect(stats.BlobsAfter)
updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed, printer.E)
updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added, printer.E)
updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
if gopts.JSON {
err := json.NewEncoder(gopts.Term.OutputWriter()).Encode(stats)
err := json.NewEncoder(globalOptions.stdout).Encode(stats)
if err != nil {
printer.E("JSON encode failed: %v", err)
Warnf("JSON encode failed: %v\n", err)
}
} else {
printer.S("")
printer.S("Files: %5d new, %5d removed, %5d changed", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
printer.S("Dirs: %5d new, %5d removed", stats.Added.Dirs, stats.Removed.Dirs)
printer.S("Others: %5d new, %5d removed", stats.Added.Others, stats.Removed.Others)
printer.S("Data Blobs: %5d new, %5d removed", stats.Added.DataBlobs, stats.Removed.DataBlobs)
printer.S("Tree Blobs: %5d new, %5d removed", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
printer.S(" Added: %-5s", ui.FormatBytes(stats.Added.Bytes))
printer.S(" Removed: %-5s", ui.FormatBytes(stats.Removed.Bytes))
Printf("\n")
Printf("Files: %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
Printf("Dirs: %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
Printf(" Added: %-5s\n", ui.FormatBytes(stats.Added.Bytes))
Printf(" Removed: %-5s\n", ui.FormatBytes(stats.Removed.Bytes))
}
return nil

View File

@@ -11,16 +11,15 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test"
)
func testRunDiffOutput(t testing.TB, gopts global.Options, firstSnapshotID string, secondSnapshotID string) (string, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) {
buf, err := withCaptureStdout(func() error {
opts := DiffOptions{
ShowMetadata: false,
}
return runDiff(ctx, opts, gopts, []string{firstSnapshotID, secondSnapshotID}, gopts.Term)
return runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
})
return buf.String(), err
}
@@ -124,10 +123,10 @@ func TestDiff(t *testing.T) {
// quiet suppresses the diff output except for the summary
env.gopts.Quiet = false
_, err := testRunDiffOutput(t, env.gopts, "", secondSnapshotID)
_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
out, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err)
for _, pattern := range diffOutputRegexPatterns {
@@ -138,7 +137,7 @@ func TestDiff(t *testing.T) {
// check quiet output
env.gopts.Quiet = true
outQuiet, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err)
rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
@@ -155,7 +154,7 @@ func TestDiffJSON(t *testing.T) {
// quiet suppresses the diff output except for the summary
env.gopts.Quiet = false
env.gopts.JSON = true
out, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err)
var stat DiffStatsContainer
@@ -182,7 +181,7 @@ func TestDiffJSON(t *testing.T) {
// check quiet output
env.gopts.Quiet = true
outQuiet, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID)
outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err)
stat = DiffStatsContainer{}

View File

@@ -7,19 +7,16 @@ import (
"path"
"path/filepath"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/dump"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newDumpCommand(globalOptions *global.Options) *cobra.Command {
func newDumpCommand() *cobra.Command {
var opts DumpOptions
cmd := &cobra.Command{
Use: "dump [flags] snapshotID file",
@@ -49,8 +46,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runDump(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
return runDump(cmd.Context(), opts, globalOptions, args)
},
}
@@ -60,7 +56,7 @@ Exit status is 12 if the password is incorrect.
// DumpOptions collects all options for the dump command.
type DumpOptions struct {
data.SnapshotFilter
restic.SnapshotFilter
Archive string
Target string
}
@@ -80,7 +76,7 @@ func splitPath(p string) []string {
return append(s, f)
}
func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
// If we print / we need to assume that there are multiple nodes at that
// level in the tree.
if pathComponents[0] == "" {
@@ -92,38 +88,35 @@ func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.
item := filepath.Join(prefix, pathComponents[0])
l := len(pathComponents)
for it := range tree {
if it.Error != nil {
return it.Error
}
for _, node := range tree.Nodes {
if ctx.Err() != nil {
return ctx.Err()
}
node := it.Node
// If dumping something in the highest level it will just take the
// first item it finds and dump that according to the switch case below.
if node.Name == pathComponents[0] {
switch {
case l == 1 && node.Type == data.NodeTypeFile:
case l == 1 && node.Type == restic.NodeTypeFile:
return d.WriteNode(ctx, node)
case l > 1 && node.Type == data.NodeTypeDir:
subtree, err := data.LoadTree(ctx, repo, *node.Subtree)
case l > 1 && node.Type == restic.NodeTypeDir:
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil {
return errors.Wrapf(err, "cannot load subtree for %q", item)
}
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
case node.Type == data.NodeTypeDir:
case node.Type == restic.NodeTypeDir:
if err := canWriteArchiveFunc(); err != nil {
return err
}
subtree, err := data.LoadTree(ctx, repo, *node.Subtree)
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil {
return err
}
return d.DumpTree(ctx, subtree, item)
case l > 1:
return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
case node.Type != data.NodeTypeFile:
case node.Type != restic.NodeTypeFile:
return fmt.Errorf("%q should be a file, but is a %q", item, node.Type)
}
}
@@ -131,13 +124,11 @@ func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.
return fmt.Errorf("path %q not found in snapshot", item)
}
func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args []string, term ui.Terminal) error {
func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatal("no file and no snapshot ID specified")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
switch opts.Archive {
case "tar", "zip":
default:
@@ -151,34 +142,39 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [
splittedPath := splitPath(path.Clean(pathToPrint))
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
defer unlock()
sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString)
sn, subfolder, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}
err = repo.LoadIndex(ctx, printer)
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
tree, err := data.LoadTree(ctx, repo, *sn.Tree)
tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
if err != nil {
return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
}
outputFileWriter := term.OutputRaw()
canWriteArchiveFunc := checkStdoutArchive(term)
outputFileWriter := os.Stdout
canWriteArchiveFunc := checkStdoutArchive
if opts.Target != "" {
file, err := os.Create(opts.Target)
@@ -202,9 +198,9 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [
return nil
}
func checkStdoutArchive(term ui.Terminal) func() error {
if term.OutputIsTerminal() {
return func() error { return fmt.Errorf("stdout is the terminal, please redirect output") }
func checkStdoutArchive() error {
if stdoutIsTerminal() {
return fmt.Errorf("stdout is the terminal, please redirect output")
}
return func() error { return nil }
return nil
}

View File

@@ -1,15 +1,16 @@
package main
import (
"fmt"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/feature"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra"
)
func newFeaturesCommand(globalOptions *global.Options) *cobra.Command {
func newFeaturesCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "features",
Short: "Print list of feature flags",
@@ -38,7 +39,7 @@ Exit status is 1 if there was any error.
return errors.Fatal("the feature command expects no arguments")
}
globalOptions.Term.Print("All Feature Flags:\n")
fmt.Printf("All Feature Flags:\n")
flags := feature.Flag.List()
tab := table.New()
@@ -50,7 +51,7 @@ Exit status is 1 if there was any error.
for _, flag := range flags {
tab.AddRow(flag)
}
return tab.Write(globalOptions.Term.OutputWriter())
return tab.Write(globalOptions.stdout)
},
}

View File

@@ -3,8 +3,6 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"sort"
"strings"
"time"
@@ -12,17 +10,14 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/walker"
)
func newFindCommand(globalOptions *global.Options) *cobra.Command {
func newFindCommand() *cobra.Command {
var opts FindOptions
cmd := &cobra.Command{
@@ -53,8 +48,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runFind(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
return runFind(cmd.Context(), opts, globalOptions, args)
},
}
@@ -73,7 +67,7 @@ type FindOptions struct {
ListLong bool
HumanReadable bool
Reverse bool
data.SnapshotFilter
restic.SnapshotFilter
}
func (opts *FindOptions) AddFlags(f *pflag.FlagSet) {
@@ -127,19 +121,13 @@ type statefulOutput struct {
HumanReadable bool
JSON bool
inuse bool
newsn *data.Snapshot
oldsn *data.Snapshot
newsn *restic.Snapshot
oldsn *restic.Snapshot
hits int
printer interface {
S(string, ...interface{})
P(string, ...interface{})
E(string, ...interface{})
}
stdout io.Writer
}
func (s *statefulOutput) PrintPatternJSON(path string, node *data.Node) {
type findNode data.Node
func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
type findNode restic.Node
b, err := json.Marshal(struct {
// Add these attributes
Path string `json:"path,omitempty"`
@@ -160,40 +148,40 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *data.Node) {
findNode: (*findNode)(node),
})
if err != nil {
s.printer.E("Marshall failed: %v", err)
Warnf("Marshall failed: %v\n", err)
return
}
if !s.inuse {
_, _ = s.stdout.Write([]byte("["))
Printf("[")
s.inuse = true
}
if s.newsn != s.oldsn {
if s.oldsn != nil {
_, _ = fmt.Fprintf(s.stdout, "],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID())
Printf("],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID())
}
_, _ = s.stdout.Write([]byte(`{"matches":[`))
Printf(`{"matches":[`)
s.oldsn = s.newsn
s.hits = 0
}
if s.hits > 0 {
_, _ = s.stdout.Write([]byte(","))
Printf(",")
}
_, _ = s.stdout.Write(b)
Print(string(b))
s.hits++
}
func (s *statefulOutput) PrintPatternNormal(path string, node *data.Node) {
func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) {
if s.newsn != s.oldsn {
if s.oldsn != nil {
s.printer.P("")
Verbosef("\n")
}
s.oldsn = s.newsn
s.printer.P("Found matching entries in snapshot %s from %s", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(global.TimeFormat))
Verbosef("Found matching entries in snapshot %s from %s\n", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(TimeFormat))
}
s.printer.S(formatNode(path, node, s.ListLong, s.HumanReadable))
Println(formatNode(path, node, s.ListLong, s.HumanReadable))
}
func (s *statefulOutput) PrintPattern(path string, node *data.Node) {
func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
if s.JSON {
s.PrintPatternJSON(path, node)
} else {
@@ -201,7 +189,7 @@ func (s *statefulOutput) PrintPattern(path string, node *data.Node) {
}
}
func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *data.Snapshot) {
func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
b, err := json.Marshal(struct {
// Add these attributes
ObjectType string `json:"object_type"`
@@ -219,32 +207,32 @@ func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *
Time: sn.Time,
})
if err != nil {
s.printer.E("Marshall failed: %v", err)
Warnf("Marshall failed: %v\n", err)
return
}
if !s.inuse {
_, _ = s.stdout.Write([]byte("["))
Printf("[")
s.inuse = true
}
if s.hits > 0 {
_, _ = s.stdout.Write([]byte(","))
Printf(",")
}
_, _ = s.stdout.Write(b)
Print(string(b))
s.hits++
}
func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *data.Snapshot) {
s.printer.S("Found %s %s", kind, id)
func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
Printf("Found %s %s\n", kind, id)
if kind == "blob" {
s.printer.S(" ... in file %s", nodepath)
s.printer.S(" (tree %s)", treeID)
Printf(" ... in file %s\n", nodepath)
Printf(" (tree %s)\n", treeID)
} else {
s.printer.S(" ... path %s", nodepath)
Printf(" ... path %s\n", nodepath)
}
s.printer.S(" ... in snapshot %s (%s)", sn.ID().Str(), sn.Time.Local().Format(global.TimeFormat))
Printf(" ... in snapshot %s (%s)\n", sn.ID().Str(), sn.Time.Local().Format(TimeFormat))
}
func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *data.Snapshot) {
func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
if s.JSON {
s.PrintObjectJSON(kind, id, nodepath, treeID, sn)
} else {
@@ -256,12 +244,12 @@ func (s *statefulOutput) Finish() {
if s.JSON {
// do some finishing up
if s.oldsn != nil {
_, _ = fmt.Fprintf(s.stdout, "],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID())
Printf("],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID())
}
if s.inuse {
_, _ = s.stdout.Write([]byte("]\n"))
Printf("]\n")
} else {
_, _ = s.stdout.Write([]byte("[]\n"))
Printf("[]\n")
}
return
}
@@ -275,14 +263,9 @@ type Finder struct {
blobIDs map[string]struct{}
treeIDs map[string]struct{}
itemsFound int
printer interface {
S(string, ...interface{})
P(string, ...interface{})
E(string, ...interface{})
}
}
func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error {
func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
if sn.Tree == nil {
@@ -290,12 +273,11 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error {
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error {
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err)
f.printer.S("Unable to load tree %s", parentTreeID)
f.printer.S(" ... which belongs to snapshot %s", sn.ID())
Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
return walker.ErrSkipNode
}
@@ -323,7 +305,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error {
}
var errIfNoMatch error
if node.Type == data.NodeTypeDir {
if node.Type == restic.NodeTypeDir {
var childMayMatch bool
for _, pat := range f.pat.pattern {
mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
@@ -381,7 +363,7 @@ func (f *Finder) findTree(treeID restic.ID, nodepath string) error {
return nil
}
func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error {
func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
debug.Log("searching IDs in snapshot %s", sn.ID())
if sn.Tree == nil {
@@ -389,12 +371,11 @@ func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error {
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error {
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err)
f.printer.S("Unable to load tree %s", parentTreeID)
f.printer.S(" ... which belongs to snapshot %s", sn.ID())
Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
return walker.ErrSkipNode
}
@@ -414,7 +395,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error {
}
}
if node.Type == data.NodeTypeFile && f.blobIDs != nil {
if node.Type == restic.NodeTypeFile && f.blobIDs != nil {
for _, id := range node.Content {
if ctx.Err() != nil {
return ctx.Err()
@@ -543,7 +524,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
for h := range indexPackIDs {
list = append(list, h)
}
f.printer.E("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
}
return packIDs, nil
}
@@ -551,20 +532,19 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
func (f *Finder) findObjectPack(id string, t restic.BlobType) {
rid, err := restic.ParseID(id)
if err != nil {
f.printer.S("Note: cannot find pack for object '%s', unable to parse ID: %v", id, err)
Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err)
return
}
blobs := f.repo.LookupBlob(t, rid)
if len(blobs) == 0 {
f.printer.S("Object %s not found in the index", rid.Str())
Printf("Object %s not found in the index\n", rid.Str())
return
}
for _, b := range blobs {
if b.ID.Equal(rid) {
f.printer.S("Object belongs to pack %s", b.PackID)
f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String())
Printf("Object belongs to pack %s\n ... Pack %s: %s\n", b.PackID, b.PackID.Str(), b.String())
break
}
}
@@ -580,13 +560,11 @@ func (f *Finder) findObjectsPacks() {
}
}
func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args []string, term ui.Terminal) error {
func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("wrong number of arguments")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
var err error
pat := findPattern{pattern: args}
if opts.CaseInsensitive {
@@ -608,10 +586,6 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
}
}
if !pat.newest.IsZero() && !pat.oldest.IsZero() && pat.oldest.After(pat.newest) {
return errors.Fatal("--oldest must specify a time before --newest")
}
// Check at most only one kind of IDs is provided: currently we
// can't mix types
if (opts.BlobID && opts.TreeID) ||
@@ -620,7 +594,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
return errors.Fatal("cannot have several ID types")
}
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
@@ -630,15 +604,15 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
if err != nil {
return err
}
if err = repo.LoadIndex(ctx, printer); err != nil {
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}
f := &Finder{
repo: repo,
pat: pat,
out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON, printer: printer, stdout: term.OutputRaw()},
printer: printer,
repo: repo,
pat: pat,
out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
}
if opts.BlobID {
@@ -661,8 +635,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
}
}
var filteredSnapshots []*data.Snapshot
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots, printer) {
var filteredSnapshots []*restic.Snapshot
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
filteredSnapshots = append(filteredSnapshots, sn)
}
if ctx.Err() != nil {

View File

@@ -7,15 +7,14 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test"
)
func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts global.Options, pattern string) []byte {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts GlobalOptions, pattern string) []byte {
buf, err := withCaptureStdout(func() error {
gopts.JSON = wantJSON
return runFind(ctx, opts, gopts, []string{pattern}, gopts.Term)
return runFind(context.TODO(), opts, gopts, []string{pattern})
})
rtest.OK(t, err)
return buf.Bytes()
@@ -96,7 +95,7 @@ func TestFindSorting(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
datafile := testSetupBackupData(t, env)
opts := BackupOptions{}
// first backup
@@ -115,14 +114,14 @@ func TestFindSorting(t *testing.T) {
// first restic find - with default FindOptions{}
results := testRunFind(t, true, FindOptions{}, env.gopts, "testfile")
lines := strings.Split(string(results), "\n")
rtest.Assert(t, len(lines) == 2, "expected two lines of output, found %d", len(lines))
rtest.Assert(t, len(lines) == 2, "expected two files found in repo (%v), found %d", datafile, len(lines))
matches := []testMatches{}
rtest.OK(t, json.Unmarshal(results, &matches))
// run second restic find with --reverse, sort oldest to newest
resultsReverse := testRunFind(t, true, FindOptions{Reverse: true}, env.gopts, "testfile")
lines = strings.Split(string(resultsReverse), "\n")
rtest.Assert(t, len(lines) == 2, "expected two lines of output, found %d", len(lines))
rtest.Assert(t, len(lines) == 2, "expected two files found in repo (%v), found %d", datafile, len(lines))
matchesReverse := []testMatches{}
rtest.OK(t, json.Unmarshal(resultsReverse, &matchesReverse))
@@ -132,12 +131,3 @@ func TestFindSorting(t *testing.T) {
rtest.Assert(t, matches[0].SnapshotID == matchesReverse[1].SnapshotID, "matches should be sorted 1")
rtest.Assert(t, matches[1].SnapshotID == matchesReverse[0].SnapshotID, "matches should be sorted 2")
}
func TestFindInvalidTimeRange(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
err := runFind(context.TODO(), FindOptions{Oldest: "2026-01-01", Newest: "2020-01-01"}, env.gopts, []string{"quack"}, env.gopts.Term)
rtest.Assert(t, err != nil && err.Error() == "Fatal: --oldest must specify a time before --newest",
"unexpected error message: %v", err)
}

View File

@@ -7,16 +7,14 @@ import (
"io"
"strconv"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newForgetCommand(globalOptions *global.Options) *cobra.Command {
func newForgetCommand() *cobra.Command {
var opts ForgetOptions
var pruneOpts PruneOptions
@@ -51,8 +49,9 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runForget(cmd.Context(), opts, pruneOpts, *globalOptions, globalOptions.Term, args)
term, cancel := setupTermstatus()
defer cancel()
return runForget(cmd.Context(), opts, pruneOpts, globalOptions, term, args)
},
}
@@ -105,21 +104,21 @@ type ForgetOptions struct {
Weekly ForgetPolicyCount
Monthly ForgetPolicyCount
Yearly ForgetPolicyCount
Within data.Duration
WithinHourly data.Duration
WithinDaily data.Duration
WithinWeekly data.Duration
WithinMonthly data.Duration
WithinYearly data.Duration
KeepTags data.TagLists
Within restic.Duration
WithinHourly restic.Duration
WithinDaily restic.Duration
WithinWeekly restic.Duration
WithinMonthly restic.Duration
WithinYearly restic.Duration
KeepTags restic.TagLists
UnsafeAllowRemoveAll bool
data.SnapshotFilter
restic.SnapshotFilter
Compact bool
// Grouping
GroupBy data.SnapshotGroupByOptions
GroupBy restic.SnapshotGroupByOptions
DryRun bool
Prune bool
}
@@ -150,7 +149,7 @@ func (opts *ForgetOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, false)
f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format")
opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true}
opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
f.BoolVar(&opts.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
@@ -164,7 +163,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
return errors.Fatal("negative values other than -1 are not allowed for --keep-*")
}
for _, d := range []data.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily,
for _, d := range []restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily,
opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} {
if d.Hours < 0 || d.Days < 0 || d.Months < 0 || d.Years < 0 {
return errors.Fatal("durations containing negative values are not allowed for --keep-within*")
@@ -174,7 +173,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
return nil
}
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts global.Options, term ui.Terminal, args []string) error {
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := verifyForgetOptions(&opts)
if err != nil {
return err
@@ -189,17 +188,22 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
if err != nil {
return err
}
defer unlock()
var snapshots data.Snapshots
verbosity := gopts.verbosity
if gopts.JSON {
verbosity = 0
}
printer := newTerminalProgressPrinter(verbosity, term)
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args, printer) {
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn)
}
if ctx.Err() != nil {
@@ -214,12 +218,12 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
removeSnIDs.Insert(*sn.ID())
}
} else {
snapshotGroups, _, err := data.GroupSnapshots(snapshots, opts.GroupBy)
snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
if err != nil {
return err
}
policy := data.ExpirePolicy{
policy := restic.ExpirePolicy{
Last: int(opts.Last),
Hourly: int(opts.Hourly),
Daily: int(opts.Daily),
@@ -254,13 +258,13 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
}
if gopts.Verbose >= 1 && !gopts.JSON {
err = PrintSnapshotGroupHeader(gopts.Term.OutputWriter(), k)
err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
if err != nil {
return err
}
}
var key data.SnapshotGroupKey
var key restic.SnapshotGroupKey
if json.Unmarshal([]byte(k), &key) != nil {
return err
}
@@ -270,25 +274,21 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
fg.Host = key.Hostname
fg.Paths = key.Paths
keep, remove, reasons := data.ApplyPolicy(snapshotGroup, policy)
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
if !policy.Empty() && len(keep) == 0 {
return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String())
}
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
printer.P("keep %d snapshots:\n", len(keep))
if err := PrintSnapshots(gopts.Term.OutputWriter(), keep, reasons, opts.Compact); err != nil {
return err
}
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
printer.P("\n")
}
fg.Keep = asJSONSnapshots(keep)
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
printer.P("remove %d snapshots:\n", len(remove))
if err := PrintSnapshots(gopts.Term.OutputWriter(), remove, nil, opts.Compact); err != nil {
return err
}
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
printer.P("\n")
}
fg.Remove = asJSONSnapshots(remove)
@@ -331,7 +331,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
}
if gopts.JSON && len(jsonGroups) > 0 {
err = printJSONForget(gopts.Term.OutputWriter(), jsonGroups)
err = printJSONForget(globalOptions.stdout, jsonGroups)
if err != nil {
return err
}
@@ -348,7 +348,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
}
pruneOptions.DryRun = opts.DryRun
return runPruneWithRepo(ctx, pruneOptions, repo, removeSnIDs, printer)
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
}
return nil
@@ -364,7 +364,7 @@ type ForgetGroup struct {
Reasons []KeepReason `json:"reasons"`
}
func asJSONSnapshots(list data.Snapshots) []Snapshot {
func asJSONSnapshots(list restic.Snapshots) []Snapshot {
var resultList []Snapshot
for _, sn := range list {
k := Snapshot{
@@ -383,7 +383,7 @@ type KeepReason struct {
Matches []string `json:"matches"`
}
func asJSONKeeps(list []data.KeepReason) []KeepReason {
func asJSONKeeps(list []restic.KeepReason) []KeepReason {
var resultList []KeepReason
for _, keep := range list {
k := KeepReason{

View File

@@ -6,22 +6,22 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunForgetMayFail(t testing.TB, gopts global.Options, opts ForgetOptions, args ...string) error {
func testRunForgetMayFail(gopts GlobalOptions, opts ForgetOptions, args ...string) error {
pruneOpts := PruneOptions{
MaxUnused: "5%",
}
return withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runForget(context.TODO(), opts, pruneOpts, gopts, gopts.Term, args)
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
})
}
func testRunForget(t testing.TB, gopts global.Options, opts ForgetOptions, args ...string) {
rtest.OK(t, testRunForgetMayFail(t, gopts, opts, args...))
func testRunForget(t testing.TB, gopts GlobalOptions, opts ForgetOptions, args ...string) {
rtest.OK(t, testRunForgetMayFail(gopts, opts, args...))
}
func TestRunForgetSafetyNet(t *testing.T) {
@@ -38,27 +38,27 @@ func TestRunForgetSafetyNet(t *testing.T) {
testListSnapshots(t, env.gopts, 2)
// --keep-tags invalid
err := testRunForgetMayFail(t, env.gopts, ForgetOptions{
KeepTags: data.TagLists{data.TagList{"invalid"}},
GroupBy: data.SnapshotGroupByOptions{Host: true, Path: true},
err := testRunForgetMayFail(env.gopts, ForgetOptions{
KeepTags: restic.TagLists{restic.TagList{"invalid"}},
GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true},
})
rtest.Assert(t, strings.Contains(err.Error(), `refusing to delete last snapshot of snapshot group "host example, path`), "wrong error message got %v", err)
// disallow `forget --unsafe-allow-remove-all`
err = testRunForgetMayFail(t, env.gopts, ForgetOptions{
err = testRunForgetMayFail(env.gopts, ForgetOptions{
UnsafeAllowRemoveAll: true,
})
rtest.Assert(t, strings.Contains(err.Error(), `--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified`), "wrong error message got %v", err)
// disallow `forget` without options
err = testRunForgetMayFail(t, env.gopts, ForgetOptions{})
err = testRunForgetMayFail(env.gopts, ForgetOptions{})
rtest.Assert(t, strings.Contains(err.Error(), `no policy was specified, no snapshots will be removed`), "wrong error message got %v", err)
// `forget --host example --unsafe-allow-remove-all` should work
testRunForget(t, env.gopts, ForgetOptions{
UnsafeAllowRemoveAll: true,
GroupBy: data.SnapshotGroupByOptions{Host: true, Path: true},
SnapshotFilter: data.SnapshotFilter{
GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true},
SnapshotFilter: restic.SnapshotFilter{
Hosts: []string{opts.Host},
},
})

View File

@@ -3,7 +3,7 @@ package main
import (
"testing"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/spf13/pflag"
)
@@ -69,18 +69,18 @@ func TestForgetOptionValues(t *testing.T) {
{ForgetOptions{Weekly: -2}, negValErrorMsg},
{ForgetOptions{Monthly: -2}, negValErrorMsg},
{ForgetOptions{Yearly: -2}, negValErrorMsg},
{ForgetOptions{Within: data.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinHourly: data.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinDaily: data.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinWeekly: data.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinMonthly: data.ParseDurationOrPanic("2y4m6d8h")}, ""},
{ForgetOptions{WithinYearly: data.ParseDurationOrPanic("2y4m6d8h")}, ""},
{ForgetOptions{Within: data.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinHourly: data.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinDaily: data.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinWeekly: data.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinMonthly: data.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg},
{ForgetOptions{WithinYearly: data.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg},
{ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""},
{ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""},
{ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg},
{ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg},
}
for _, testCase := range testCases {
@@ -96,38 +96,7 @@ func TestForgetOptionValues(t *testing.T) {
func TestForgetHostnameDefaulting(t *testing.T) {
t.Setenv("RESTIC_HOST", "testhost")
tests := []struct {
name string
args []string
want []string
}{
{
name: "env default when flag not set",
args: nil,
want: []string{"testhost"},
},
{
name: "flag overrides env",
args: []string{"--host", "flaghost"},
want: []string{"flaghost"},
},
{
name: "empty flag clears env",
args: []string{"--host", ""},
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
set := pflag.NewFlagSet(tt.name, pflag.ContinueOnError)
opts := ForgetOptions{}
opts.AddFlags(set)
err := set.Parse(tt.args)
rtest.Assert(t, err == nil, "expected no error for input")
finalizeSnapshotFilter(&opts.SnapshotFilter)
rtest.Equals(t, tt.want, opts.Hosts)
})
}
opts := ForgetOptions{}
opts.AddFlags(pflag.NewFlagSet("test", pflag.ContinueOnError))
rtest.Equals(t, []string{"testhost"}, opts.Hosts)
}

View File

@@ -6,15 +6,12 @@ import (
"time"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"github.com/spf13/pflag"
)
func newGenerateCommand(globalOptions *global.Options) *cobra.Command {
func newGenerateCommand() *cobra.Command {
var opts generateOptions
cmd := &cobra.Command{
@@ -32,7 +29,7 @@ Exit status is 1 if there was any error.
`,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return runGenerate(opts, *globalOptions, args, globalOptions.Term)
return runGenerate(opts, args)
},
}
opts.AddFlags(cmd.Flags())
@@ -55,7 +52,7 @@ func (opts *generateOptions) AddFlags(f *pflag.FlagSet) {
f.StringVar(&opts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file` (`-` for stdout)")
}
func writeManpages(root *cobra.Command, dir string, printer progress.Printer) error {
func writeManpages(root *cobra.Command, dir string) error {
// use a fixed date for the man pages so that generating them is deterministic
date, err := time.Parse("Jan 2006", "Jan 2017")
if err != nil {
@@ -69,12 +66,14 @@ func writeManpages(root *cobra.Command, dir string, printer progress.Printer) er
Date: &date,
}
printer.P("writing man pages to directory %v", dir)
Verbosef("writing man pages to directory %v\n", dir)
return doc.GenManTree(root, header, dir)
}
func writeCompletion(filename string, shell string, generate func(w io.Writer) error, printer progress.Printer, gopts global.Options) (err error) {
printer.PT("writing %s completion file to %v", shell, filename)
func writeCompletion(filename string, shell string, generate func(w io.Writer) error) (err error) {
if stdoutIsTerminal() {
Verbosef("writing %s completion file to %v\n", shell, filename)
}
var outWriter io.Writer
if filename != "-" {
var outFile *os.File
@@ -85,7 +84,7 @@ func writeCompletion(filename string, shell string, generate func(w io.Writer) e
defer func() { err = outFile.Close() }()
outWriter = outFile
} else {
outWriter = gopts.Term.OutputWriter()
outWriter = globalOptions.stdout
}
err = generate(outWriter)
@@ -111,16 +110,15 @@ func checkStdoutForSingleShell(opts generateOptions) error {
return nil
}
func runGenerate(opts generateOptions, gopts global.Options, args []string, term ui.Terminal) error {
func runGenerate(opts generateOptions, args []string) error {
if len(args) > 0 {
return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
cmdRoot := newRootCommand(&global.Options{})
cmdRoot := newRootCommand()
if opts.ManDir != "" {
err := writeManpages(cmdRoot, opts.ManDir, printer)
err := writeManpages(cmdRoot, opts.ManDir)
if err != nil {
return err
}
@@ -132,28 +130,28 @@ func runGenerate(opts generateOptions, gopts global.Options, args []string, term
}
if opts.BashCompletionFile != "" {
err := writeCompletion(opts.BashCompletionFile, "bash", cmdRoot.GenBashCompletion, printer, gopts)
err := writeCompletion(opts.BashCompletionFile, "bash", cmdRoot.GenBashCompletion)
if err != nil {
return err
}
}
if opts.FishCompletionFile != "" {
err := writeCompletion(opts.FishCompletionFile, "fish", func(w io.Writer) error { return cmdRoot.GenFishCompletion(w, true) }, printer, gopts)
err := writeCompletion(opts.FishCompletionFile, "fish", func(w io.Writer) error { return cmdRoot.GenFishCompletion(w, true) })
if err != nil {
return err
}
}
if opts.ZSHCompletionFile != "" {
err := writeCompletion(opts.ZSHCompletionFile, "zsh", cmdRoot.GenZshCompletion, printer, gopts)
err := writeCompletion(opts.ZSHCompletionFile, "zsh", cmdRoot.GenZshCompletion)
if err != nil {
return err
}
}
if opts.PowerShellCompletionFile != "" {
err := writeCompletion(opts.PowerShellCompletionFile, "powershell", cmdRoot.GenPowerShellCompletion, printer, gopts)
err := writeCompletion(opts.PowerShellCompletionFile, "powershell", cmdRoot.GenPowerShellCompletion)
if err != nil {
return err
}

View File

@@ -1,21 +1,13 @@
package main
import (
"context"
"bytes"
"strings"
"testing"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test"
)
func testRunGenerate(t testing.TB, gopts global.Options, opts generateOptions) ([]byte, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runGenerate(opts, gopts, []string{}, gopts.Term)
})
return buf.Bytes(), err
}
func TestGenerateStdout(t *testing.T) {
testCases := []struct {
name string
@@ -29,14 +21,20 @@ func TestGenerateStdout(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
output, err := testRunGenerate(t, global.Options{}, tc.opts)
buf := bytes.NewBuffer(nil)
globalOptions.stdout = buf
err := runGenerate(tc.opts, []string{})
rtest.OK(t, err)
rtest.Assert(t, strings.Contains(string(output), "# "+tc.name+" completion for restic"), "has no expected completion header")
completionString := buf.String()
rtest.Assert(t, strings.Contains(completionString, "# "+tc.name+" completion for restic"), "has no expected completion header")
})
}
t.Run("Generate shell completions to stdout for two shells", func(t *testing.T) {
_, err := testRunGenerate(t, global.Options{}, generateOptions{BashCompletionFile: "-", FishCompletionFile: "-"})
buf := bytes.NewBuffer(nil)
globalOptions.stdout = buf
opts := generateOptions{BashCompletionFile: "-", FishCompletionFile: "-"}
err := runGenerate(opts, []string{})
rtest.Assert(t, err != nil, "generate shell completions to stdout for two shells fails")
})
}

View File

@@ -8,16 +8,14 @@ import (
"github.com/restic/chunker"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newInitCommand(globalOptions *global.Options) *cobra.Command {
func newInitCommand() *cobra.Command {
var opts InitOptions
cmd := &cobra.Command{
@@ -35,7 +33,7 @@ Exit status is 1 if there was any error.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runInit(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
return runInit(cmd.Context(), opts, globalOptions, args)
},
}
opts.AddFlags(cmd.Flags())
@@ -44,78 +42,105 @@ Exit status is 1 if there was any error.
// InitOptions bundles all options for the init command.
type InitOptions struct {
global.SecondaryRepoOptions
secondaryRepoOptions
CopyChunkerParameters bool
RepositoryVersion string
}
func (opts *InitOptions) AddFlags(f *pflag.FlagSet) {
opts.SecondaryRepoOptions.AddFlags(f, "secondary", "to copy chunker parameters from")
opts.secondaryRepoOptions.AddFlags(f, "secondary", "to copy chunker parameters from")
f.BoolVar(&opts.CopyChunkerParameters, "copy-chunker-params", false, "copy chunker parameters from the secondary repository (useful with the copy command)")
f.StringVar(&opts.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
}
func runInit(ctx context.Context, opts InitOptions, gopts global.Options, args []string, term ui.Terminal) error {
func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {
if len(args) > 0 {
return errors.Fatal("the init command expects no arguments, only options - please see `restic help init` for usage and flags")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
var version uint
switch opts.RepositoryVersion {
case "latest", "":
if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
version = restic.MaxRepoVersion
case "stable":
} else if opts.RepositoryVersion == "stable" {
version = restic.StableRepoVersion
default:
} else {
v, err := strconv.ParseUint(opts.RepositoryVersion, 10, 32)
if err != nil {
return errors.Fatal("invalid repository version")
}
version = uint(v)
}
if version < restic.MinRepoVersion || version > restic.MaxRepoVersion {
return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
}
chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts, printer)
chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts)
if err != nil {
return err
}
s, err := global.CreateRepository(ctx, gopts, version, chunkerPolynomial, printer)
gopts.Repo, err = ReadRepo(gopts)
if err != nil {
return errors.Fatalf("%s", err)
return err
}
gopts.password, err = ReadPasswordTwice(ctx, gopts,
"enter password for new repository: ",
"enter password again: ")
if err != nil {
return err
}
be, err := create(ctx, gopts.Repo, gopts, gopts.extended)
if err != nil {
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
}
s, err := repository.New(be, repository.Options{
Compression: gopts.Compression,
PackSize: gopts.PackSize * 1024 * 1024,
})
if err != nil {
return errors.Fatal(err.Error())
}
err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
if err != nil {
return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
}
if !gopts.JSON {
printer.P("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.Backends, gopts.Repo))
Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.backends, gopts.Repo))
if opts.CopyChunkerParameters && chunkerPolynomial != nil {
printer.P(" with chunker parameters copied from secondary repository")
Verbosef(" with chunker parameters copied from secondary repository\n")
} else {
Verbosef("\n")
}
printer.P("")
printer.P("Please note that knowledge of your password is required to access")
printer.P("the repository. Losing your password means that your data is")
printer.P("irrecoverably lost.")
Verbosef("\n")
Verbosef("Please note that knowledge of your password is required to access\n")
Verbosef("the repository. Losing your password means that your data is\n")
Verbosef("irrecoverably lost.\n")
} else {
status := initSuccess{
MessageType: "initialized",
ID: s.Config().ID,
Repository: location.StripPassword(gopts.Backends, gopts.Repo),
Repository: location.StripPassword(gopts.backends, gopts.Repo),
}
return json.NewEncoder(gopts.Term.OutputWriter()).Encode(status)
return json.NewEncoder(globalOptions.stdout).Encode(status)
}
return nil
}
func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts global.Options, printer progress.Printer) (*chunker.Pol, error) {
func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
if opts.CopyChunkerParameters {
otherGopts, _, err := opts.SecondaryRepoOptions.FillGlobalOpts(ctx, gopts, "secondary")
otherGopts, _, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "secondary")
if err != nil {
return nil, err
}
otherRepo, err := global.OpenRepository(ctx, otherGopts, printer)
otherRepo, err := OpenRepository(ctx, otherGopts)
if err != nil {
return nil, err
}

View File

@@ -6,27 +6,22 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/progress"
)
func testRunInit(t testing.TB, gopts global.Options) {
func testRunInit(t testing.TB, opts GlobalOptions) {
repository.TestUseLowSecurityKDFParameters(t)
restic.TestDisableCheckPolynomial(t)
restic.TestSetLockTimeout(t, 0)
err := withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runInit(ctx, InitOptions{}, gopts, nil, gopts.Term)
})
rtest.OK(t, err)
t.Logf("repository initialized at %v", gopts.Repo)
rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil))
t.Logf("repository initialized at %v", opts.Repo)
// create temporary junk files to verify that restic does not trip over them
for _, path := range []string{"index", "snapshots", "keys", "locks", filepath.Join("data", "00")} {
rtest.OK(t, os.WriteFile(filepath.Join(gopts.Repo, path, "tmp12345"), []byte("junk file"), 0o600))
rtest.OK(t, os.WriteFile(filepath.Join(opts.Repo, path, "tmp12345"), []byte("junk file"), 0o600))
}
}
@@ -39,34 +34,20 @@ func TestInitCopyChunkerParams(t *testing.T) {
testRunInit(t, env2.gopts)
initOpts := InitOptions{
SecondaryRepoOptions: global.SecondaryRepoOptions{
secondaryRepoOptions: secondaryRepoOptions{
Repo: env2.gopts.Repo,
Password: env2.gopts.Password,
password: env2.gopts.password,
},
}
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runInit(ctx, initOpts, gopts, nil, gopts.Term)
})
rtest.Assert(t, err != nil, "expected invalid init options to fail")
rtest.Assert(t, runInit(context.TODO(), initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
initOpts.CopyChunkerParameters = true
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runInit(ctx, initOpts, gopts, nil, gopts.Term)
})
rtest.OK(t, runInit(context.TODO(), initOpts, env.gopts, nil))
repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
var repo *repository.Repository
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
repo, err = global.OpenRepository(ctx, gopts, &progress.NoopPrinter{})
return err
})
rtest.OK(t, err)
var otherRepo *repository.Repository
err = withTermStatus(t, env2.gopts, func(ctx context.Context, gopts global.Options) error {
otherRepo, err = global.OpenRepository(ctx, gopts, &progress.NoopPrinter{})
return err
})
otherRepo, err := OpenRepository(context.TODO(), env2.gopts)
rtest.OK(t, err)
rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial,

View File

@@ -1,11 +1,10 @@
package main
import (
"github.com/restic/restic/internal/global"
"github.com/spf13/cobra"
)
func newKeyCommand(globalOptions *global.Options) *cobra.Command {
func newKeyCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "key",
Short: "Manage keys (passwords)",
@@ -18,10 +17,10 @@ per repository.
}
cmd.AddCommand(
newKeyAddCommand(globalOptions),
newKeyListCommand(globalOptions),
newKeyPasswdCommand(globalOptions),
newKeyRemoveCommand(globalOptions),
newKeyAddCommand(),
newKeyListCommand(),
newKeyPasswdCommand(),
newKeyRemoveCommand(),
)
return cmd
}

View File

@@ -5,15 +5,12 @@ import (
"fmt"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newKeyAddCommand(globalOptions *global.Options) *cobra.Command {
func newKeyAddCommand() *cobra.Command {
var opts KeyAddOptions
cmd := &cobra.Command{
@@ -33,7 +30,7 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runKeyAdd(cmd.Context(), *globalOptions, opts, args, globalOptions.Term)
return runKeyAdd(cmd.Context(), globalOptions, opts, args)
},
}
@@ -55,22 +52,21 @@ func (opts *KeyAddOptions) Add(flags *pflag.FlagSet) {
flags.StringVarP(&opts.Hostname, "host", "", "", "the hostname for new key")
}
func runKeyAdd(ctx context.Context, gopts global.Options, opts KeyAddOptions, args []string, term ui.Terminal) error {
func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error {
if len(args) > 0 {
return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags")
}
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false, printer)
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
if err != nil {
return err
}
defer unlock()
return addKey(ctx, repo, gopts, opts, printer)
return addKey(ctx, repo, gopts, opts)
}
func addKey(ctx context.Context, repo *repository.Repository, gopts global.Options, opts KeyAddOptions, printer progress.Printer) error {
func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error {
pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword)
if err != nil {
return err
@@ -78,7 +74,7 @@ func addKey(ctx context.Context, repo *repository.Repository, gopts global.Optio
id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v", err)
return errors.Fatalf("creating new key failed: %v\n", err)
}
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
@@ -86,7 +82,7 @@ func addKey(ctx context.Context, repo *repository.Repository, gopts global.Optio
return err
}
printer.P("saved new key with ID %s", id.ID())
Verbosef("saved new key with ID %s\n", id.ID())
return nil
}
@@ -94,7 +90,7 @@ func addKey(ctx context.Context, repo *repository.Repository, gopts global.Optio
// testKeyNewPassword is used to set a new password during integration testing.
var testKeyNewPassword string
func getNewPassword(ctx context.Context, gopts global.Options, newPasswordFile string, insecureNoPassword bool) (string, error) {
func getNewPassword(ctx context.Context, gopts GlobalOptions, newPasswordFile string, insecureNoPassword bool) (string, error) {
if testKeyNewPassword != "" {
return testKeyNewPassword, nil
}
@@ -107,7 +103,7 @@ func getNewPassword(ctx context.Context, gopts global.Options, newPasswordFile s
}
if newPasswordFile != "" {
password, err := global.LoadPasswordFromFile(newPasswordFile)
password, err := loadPasswordFromFile(newPasswordFile)
if err != nil {
return "", err
}
@@ -120,11 +116,11 @@ func getNewPassword(ctx context.Context, gopts global.Options, newPasswordFile s
// Since we already have an open repository, temporary remove the password
// to prompt the user for the passwd.
newopts := gopts
newopts.Password = ""
newopts.password = ""
// empty passwords are already handled above
newopts.InsecureNoPassword = false
return global.ReadPasswordTwice(ctx, newopts,
return ReadPasswordTwice(ctx, newopts,
"enter new password: ",
"enter password again: ")
}

View File

@@ -10,15 +10,13 @@ import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/progress"
)
func testRunKeyListOtherIDs(t testing.TB, gopts global.Options) []string {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyList(ctx, gopts, []string{}, gopts.Term)
func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
buf, err := withCaptureStdout(func() error {
return runKeyList(context.TODO(), gopts, []string{})
})
rtest.OK(t, err)
@@ -35,64 +33,49 @@ func testRunKeyListOtherIDs(t testing.TB, gopts global.Options) []string {
return IDs
}
func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts global.Options) {
func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions) {
testKeyNewPassword = newPassword
defer func() {
testKeyNewPassword = ""
}()
err := withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyAdd(ctx, gopts, KeyAddOptions{}, []string{}, gopts.Term)
})
rtest.OK(t, err)
rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{}, []string{}))
}
func testRunKeyAddNewKeyUserHost(t testing.TB, gopts global.Options) {
func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
testKeyNewPassword = "john's geheimnis"
defer func() {
testKeyNewPassword = ""
}()
t.Log("adding key for john@example.com")
err := withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyAdd(ctx, gopts, KeyAddOptions{
Username: "john",
Hostname: "example.com",
}, []string{}, gopts.Term)
})
rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{
Username: "john",
Hostname: "example.com",
}, []string{}))
repo, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
key, err := repository.SearchKey(context.TODO(), repo, testKeyNewPassword, 2, "")
rtest.OK(t, err)
_ = withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
repo, err := global.OpenRepository(ctx, gopts, &progress.NoopPrinter{})
rtest.OK(t, err)
key, err := repository.SearchKey(ctx, repo, testKeyNewPassword, 2, "")
rtest.OK(t, err)
rtest.Equals(t, "john", key.Username)
rtest.Equals(t, "example.com", key.Hostname)
return nil
})
rtest.Equals(t, "john", key.Username)
rtest.Equals(t, "example.com", key.Hostname)
}
func testRunKeyPasswd(t testing.TB, newPassword string, gopts global.Options) {
func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
testKeyNewPassword = newPassword
defer func() {
testKeyNewPassword = ""
}()
err := withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyPasswd(ctx, gopts, KeyPasswdOptions{}, []string{}, gopts.Term)
})
rtest.OK(t, err)
rtest.OK(t, runKeyPasswd(context.TODO(), gopts, KeyPasswdOptions{}, []string{}))
}
func testRunKeyRemove(t testing.TB, gopts global.Options, IDs []string) {
func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
t.Logf("remove %d keys: %q\n", len(IDs), IDs)
for _, id := range IDs {
err := withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyRemove(ctx, gopts, []string{id}, gopts.Term)
})
rtest.OK(t, err)
rtest.OK(t, runKeyRemove(context.TODO(), gopts, []string{id}))
}
}
@@ -104,28 +87,25 @@ func TestKeyAddRemove(t *testing.T) {
env, cleanup := withTestEnvironment(t)
// must list keys more than once
env.gopts.BackendTestHook = nil
env.gopts.backendTestHook = nil
defer cleanup()
testRunInit(t, env.gopts)
testRunKeyPasswd(t, "geheim2", env.gopts)
env.gopts.Password = "geheim2"
t.Logf("changed password to %q", env.gopts.Password)
env.gopts.password = "geheim2"
t.Logf("changed password to %q", env.gopts.password)
for _, newPassword := range passwordList {
testRunKeyAddNewKey(t, newPassword, env.gopts)
t.Logf("added new password %q", newPassword)
env.gopts.Password = newPassword
env.gopts.password = newPassword
testRunKeyRemove(t, env.gopts, testRunKeyListOtherIDs(t, env.gopts))
}
env.gopts.Password = passwordList[len(passwordList)-1]
t.Logf("testing access with last password %q\n", env.gopts.Password)
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyList(ctx, gopts, []string{}, gopts.Term)
})
rtest.OK(t, err)
env.gopts.password = passwordList[len(passwordList)-1]
t.Logf("testing access with last password %q\n", env.gopts.password)
rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
testRunCheck(t, env.gopts)
testRunKeyAddNewKeyUserHost(t, env.gopts)
@@ -136,40 +116,33 @@ func TestKeyAddInvalid(t *testing.T) {
defer cleanup()
testRunInit(t, env.gopts)
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyAdd(ctx, gopts, KeyAddOptions{
NewPasswordFile: "some-file",
InsecureNoPassword: true,
}, []string{}, gopts.Term)
})
err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
NewPasswordFile: "some-file",
InsecureNoPassword: true,
}, []string{})
rtest.Assert(t, strings.Contains(err.Error(), "only either"), "unexpected error message, got %q", err)
pwfile := filepath.Join(t.TempDir(), "pwfile")
rtest.OK(t, os.WriteFile(pwfile, []byte{}, 0o666))
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyAdd(ctx, gopts, KeyAddOptions{
NewPasswordFile: pwfile,
}, []string{}, gopts.Term)
})
err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
NewPasswordFile: pwfile,
}, []string{})
rtest.Assert(t, strings.Contains(err.Error(), "an empty password is not allowed by default"), "unexpected error message, got %q", err)
}
func TestKeyAddEmpty(t *testing.T) {
env, cleanup := withTestEnvironment(t)
// must list keys more than once
env.gopts.BackendTestHook = nil
env.gopts.backendTestHook = nil
defer cleanup()
testRunInit(t, env.gopts)
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyAdd(ctx, gopts, KeyAddOptions{
InsecureNoPassword: true,
}, []string{}, gopts.Term)
})
rtest.OK(t, err)
rtest.OK(t, runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
InsecureNoPassword: true,
}, []string{}))
env.gopts.Password = ""
env.gopts.password = ""
env.gopts.InsecureNoPassword = true
testRunCheck(t, env.gopts)
@@ -188,7 +161,7 @@ func TestKeyProblems(t *testing.T) {
defer cleanup()
testRunInit(t, env.gopts)
env.gopts.BackendTestHook = func(r backend.Backend) (backend.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &emptySaveBackend{r}, nil
}
@@ -197,23 +170,16 @@ func TestKeyProblems(t *testing.T) {
testKeyNewPassword = ""
}()
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyPasswd(ctx, gopts, KeyPasswdOptions{}, []string{}, gopts.Term)
})
err := runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{})
t.Log(err)
rtest.Assert(t, err != nil, "expected passwd change to fail")
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyAdd(ctx, gopts, KeyAddOptions{}, []string{}, gopts.Term)
})
err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{})
t.Log(err)
rtest.Assert(t, err != nil, "expected key adding to fail")
t.Logf("testing access with initial password %q\n", env.gopts.Password)
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyList(ctx, gopts, []string{}, gopts.Term)
})
rtest.OK(t, err)
t.Logf("testing access with initial password %q\n", env.gopts.password)
rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
testRunCheck(t, env.gopts)
}
@@ -222,37 +188,27 @@ func TestKeyCommandInvalidArguments(t *testing.T) {
defer cleanup()
testRunInit(t, env.gopts)
env.gopts.BackendTestHook = func(r backend.Backend) (backend.Backend, error) {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &emptySaveBackend{r}, nil
}
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyAdd(ctx, gopts, KeyAddOptions{}, []string{"johndoe"}, gopts.Term)
})
err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"})
t.Log(err)
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key add: %v", err)
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyPasswd(ctx, gopts, KeyPasswdOptions{}, []string{"johndoe"}, gopts.Term)
})
err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"})
t.Log(err)
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key passwd: %v", err)
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyList(ctx, gopts, []string{"johndoe"}, gopts.Term)
})
err = runKeyList(context.TODO(), env.gopts, []string{"johndoe"})
t.Log(err)
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key list: %v", err)
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyRemove(ctx, gopts, []string{}, gopts.Term)
})
err = runKeyRemove(context.TODO(), env.gopts, []string{})
t.Log(err)
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runKeyRemove(ctx, gopts, []string{"john", "doe"}, gopts.Term)
})
err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"})
t.Log(err)
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
}

View File

@@ -6,16 +6,13 @@ import (
"fmt"
"sync"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra"
)
func newKeyListCommand(globalOptions *global.Options) *cobra.Command {
func newKeyListCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List keys (passwords)",
@@ -35,28 +32,27 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runKeyList(cmd.Context(), *globalOptions, args, globalOptions.Term)
return runKeyList(cmd.Context(), globalOptions, args)
},
}
return cmd
}
func runKeyList(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) > 0 {
return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
defer unlock()
return listKeys(ctx, repo, gopts, printer)
return listKeys(ctx, repo, gopts)
}
func listKeys(ctx context.Context, s *repository.Repository, gopts global.Options, printer progress.Printer) error {
func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
type keyInfo struct {
Current bool `json:"current"`
ID string `json:"id"`
@@ -72,7 +68,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts global.Option
err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, _ int64) error {
k, err := repository.LoadKey(ctx, s, id)
if err != nil {
printer.E("LoadKey() failed: %v", err)
Warnf("LoadKey() failed: %v\n", err)
return nil
}
@@ -82,7 +78,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts global.Option
ShortID: id.Str(),
UserName: k.Username,
HostName: k.Hostname,
Created: k.Created.Local().Format(global.TimeFormat),
Created: k.Created.Local().Format(TimeFormat),
}
m.Lock()
@@ -96,7 +92,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts global.Option
}
if gopts.JSON {
return json.NewEncoder(gopts.Term.OutputWriter()).Encode(keys)
return json.NewEncoder(globalOptions.stdout).Encode(keys)
}
tab := table.New()
@@ -109,5 +105,5 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts global.Option
tab.AddRow(key)
}
return tab.Write(gopts.Term.OutputWriter())
return tab.Write(globalOptions.stdout)
}

View File

@@ -5,15 +5,12 @@ import (
"fmt"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newKeyPasswdCommand(globalOptions *global.Options) *cobra.Command {
func newKeyPasswdCommand() *cobra.Command {
var opts KeyPasswdOptions
cmd := &cobra.Command{
@@ -34,7 +31,7 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runKeyPasswd(cmd.Context(), *globalOptions, opts, args, globalOptions.Term)
return runKeyPasswd(cmd.Context(), globalOptions, opts, args)
},
}
@@ -50,22 +47,21 @@ func (opts *KeyPasswdOptions) AddFlags(flags *pflag.FlagSet) {
opts.KeyAddOptions.Add(flags)
}
func runKeyPasswd(ctx context.Context, gopts global.Options, opts KeyPasswdOptions, args []string, term ui.Terminal) error {
func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error {
if len(args) > 0 {
return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags")
}
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if err != nil {
return err
}
defer unlock()
return changePassword(ctx, repo, gopts, opts, printer)
return changePassword(ctx, repo, gopts, opts)
}
func changePassword(ctx context.Context, repo *repository.Repository, gopts global.Options, opts KeyPasswdOptions, printer progress.Printer) error {
func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error {
pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword)
if err != nil {
return err
@@ -73,7 +69,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts glob
id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v", err)
return errors.Fatalf("creating new key failed: %v\n", err)
}
oldID := repo.KeyID()
@@ -87,7 +83,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts glob
return err
}
printer.P("saved new key as %s", id)
Verbosef("saved new key as %s\n", id)
return nil
}

View File

@@ -5,15 +5,12 @@ import (
"fmt"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/spf13/cobra"
)
func newKeyRemoveCommand(globalOptions *global.Options) *cobra.Command {
func newKeyRemoveCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "remove [ID]",
Short: "Remove key ID (password) from the repository.",
@@ -32,28 +29,27 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runKeyRemove(cmd.Context(), *globalOptions, args, globalOptions.Term)
return runKeyRemove(cmd.Context(), globalOptions, args)
},
}
return cmd
}
func runKeyRemove(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
return fmt.Errorf("key remove expects one argument as the key id")
}
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if err != nil {
return err
}
defer unlock()
return deleteKey(ctx, repo, args[0], printer)
return deleteKey(ctx, repo, args[0])
}
func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string, printer progress.Printer) error {
func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error {
id, err := restic.Find(ctx, repo, restic.KeyFile, idPrefix)
if err != nil {
return err
@@ -68,6 +64,6 @@ func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string
return err
}
printer.P("removed key %v", id)
Verbosef("removed key %v\n", id)
return nil
}

View File

@@ -5,15 +5,13 @@ import (
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/spf13/cobra"
)
func newListCommand(globalOptions *global.Options) *cobra.Command {
func newListCommand() *cobra.Command {
var listAllowedArgs = []string{"blobs", "packs", "index", "snapshots", "keys", "locks"}
var listAllowedArgsUseString = strings.Join(listAllowedArgs, "|")
@@ -35,7 +33,7 @@ Exit status is 12 if the password is incorrect.
DisableAutoGenTag: true,
GroupID: cmdGroupDefault,
RunE: func(cmd *cobra.Command, args []string) error {
return runList(cmd.Context(), *globalOptions, args, globalOptions.Term)
return runList(cmd.Context(), globalOptions, args)
},
ValidArgs: listAllowedArgs,
Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
@@ -43,14 +41,12 @@ Exit status is 12 if the password is incorrect.
return cmd
}
func runList(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified")
}
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock || args[0] == "locks", printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock || args[0] == "locks")
if err != nil {
return err
}
@@ -73,20 +69,16 @@ func runList(ctx context.Context, gopts global.Options, args []string, term ui.T
if err != nil {
return err
}
for blobs := range idx.Values() {
if ctx.Err() != nil {
return ctx.Err()
}
printer.S("%v %v", blobs.Type, blobs.ID)
}
return nil
return idx.Each(ctx, func(blobs restic.PackedBlob) {
Printf("%v %v\n", blobs.Type, blobs.ID)
})
})
default:
return errors.Fatal("invalid type")
}
return repo.List(ctx, t, func(id restic.ID, _ int64) error {
printer.S("%s", id)
Printf("%s\n", id)
return nil
})
}

View File

@@ -4,19 +4,15 @@ import (
"bufio"
"context"
"io"
"path/filepath"
"strings"
"testing"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
)
func testRunList(t testing.TB, gopts global.Options, tpe string) restic.IDs {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runList(ctx, gopts, []string{tpe}, gopts.Term)
func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
buf, err := withCaptureStdout(func() error {
return runList(context.TODO(), opts, []string{tpe})
})
rtest.OK(t, err)
return parseIDsFromReader(t, buf)
@@ -28,77 +24,21 @@ func parseIDsFromReader(t testing.TB, rd io.Reader) restic.IDs {
sc := bufio.NewScanner(rd)
for sc.Scan() {
if len(sc.Text()) == 64 {
id, err := restic.ParseID(sc.Text())
if err != nil {
t.Logf("parse id %v: %v", sc.Text(), err)
continue
}
IDs = append(IDs, id)
} else {
// 'list blobs' is different because it lists the blobs together with the blob type
// e.g. "tree ac08ce34ba4f8123618661bef2425f7028ffb9ac740578a3ee88684d2523fee8"
parts := strings.Split(sc.Text(), " ")
id, err := restic.ParseID(parts[len(parts)-1])
if err != nil {
t.Logf("parse id %v: %v", sc.Text(), err)
continue
}
IDs = append(IDs, id)
id, err := restic.ParseID(sc.Text())
if err != nil {
t.Logf("parse id %v: %v", sc.Text(), err)
continue
}
IDs = append(IDs, id)
}
return IDs
}
func testListSnapshots(t testing.TB, gopts global.Options, expected int) restic.IDs {
func testListSnapshots(t testing.TB, opts GlobalOptions, expected int) restic.IDs {
t.Helper()
snapshotIDs := testRunList(t, gopts, "snapshots")
snapshotIDs := testRunList(t, "snapshots", opts)
rtest.Assert(t, len(snapshotIDs) == expected, "expected %v snapshot, got %v", expected, snapshotIDs)
return snapshotIDs
}
// extract blob set from repository index
func testListBlobs(t testing.TB, gopts global.Options) (blobSetFromIndex restic.IDSet) {
err := withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
rtest.OK(t, err)
defer unlock()
// make sure the index is loaded
rtest.OK(t, repo.LoadIndex(ctx, nil))
// get blobs from index
blobSetFromIndex = restic.NewIDSet()
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
blobSetFromIndex.Insert(blob.ID)
}))
return nil
})
rtest.OK(t, err)
return blobSetFromIndex
}
func TestListBlobs(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
opts := BackupOptions{}
// first backup
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
testListSnapshots(t, env.gopts, 1)
// run the `list blobs` command
resticIDs := testRunList(t, env.gopts, "blobs")
// convert to set
testIDSet := restic.NewIDSet(resticIDs...)
blobSetFromIndex := testListBlobs(t, env.gopts)
rtest.Assert(t, blobSetFromIndex.Equals(testIDSet), "the set of restic.ID s should be equal")
}

View File

@@ -15,16 +15,13 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/walker"
)
func newLsCommand(globalOptions *global.Options) *cobra.Command {
func newLsCommand() *cobra.Command {
var opts LsOptions
cmd := &cobra.Command{
@@ -62,8 +59,7 @@ Exit status is 12 if the password is incorrect.
DisableAutoGenTag: true,
GroupID: cmdGroupDefault,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runLs(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
return runLs(cmd.Context(), opts, globalOptions, args)
},
}
opts.AddFlags(cmd.Flags())
@@ -73,7 +69,7 @@ Exit status is 12 if the password is incorrect.
// LsOptions collects all options for the ls command.
type LsOptions struct {
ListLong bool
data.SnapshotFilter
restic.SnapshotFilter
Recursive bool
HumanReadable bool
Ncdu bool
@@ -92,8 +88,8 @@ func (opts *LsOptions) AddFlags(f *pflag.FlagSet) {
}
type lsPrinter interface {
Snapshot(sn *data.Snapshot) error
Node(path string, node *data.Node, isPrefixDirectory bool) error
Snapshot(sn *restic.Snapshot) error
Node(path string, node *restic.Node, isPrefixDirectory bool) error
LeaveDir(path string) error
Close() error
}
@@ -102,9 +98,9 @@ type jsonLsPrinter struct {
enc *json.Encoder
}
func (p *jsonLsPrinter) Snapshot(sn *data.Snapshot) error {
func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) error {
type lsSnapshot struct {
*data.Snapshot
*restic.Snapshot
ID *restic.ID `json:"id"`
ShortID string `json:"short_id"` // deprecated
MessageType string `json:"message_type"` // "snapshot"
@@ -121,14 +117,14 @@ func (p *jsonLsPrinter) Snapshot(sn *data.Snapshot) error {
}
// Node formats node in our custom JSON format, followed by a newline.
func (p *jsonLsPrinter) Node(path string, node *data.Node, isPrefixDirectory bool) error {
func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
if isPrefixDirectory {
return nil
}
return lsNodeJSON(p.enc, path, node)
}
func lsNodeJSON(enc *json.Encoder, path string, node *data.Node) error {
func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
n := &struct {
Name string `json:"name"`
Type string `json:"type"`
@@ -164,7 +160,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *data.Node) error {
}
// Always print size for regular files, even when empty,
// but never for other types.
if node.Type == data.NodeTypeFile {
if node.Type == restic.NodeTypeFile {
n.Size = &n.size
}
@@ -182,7 +178,7 @@ type ncduLsPrinter struct {
// Snapshot prints a restic snapshot in Ncdu save format.
// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
func (p *ncduLsPrinter) Snapshot(sn *data.Snapshot) error {
func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) error {
const NcduMajorVer = 1
const NcduMinorVer = 2
@@ -195,7 +191,7 @@ func (p *ncduLsPrinter) Snapshot(sn *data.Snapshot) error {
return err
}
func lsNcduNode(_ string, node *data.Node) ([]byte, error) {
func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
type NcduNode struct {
Name string `json:"name"`
Asize uint64 `json:"asize"`
@@ -220,7 +216,7 @@ func lsNcduNode(_ string, node *data.Node) ([]byte, error) {
Dev: node.DeviceID,
Ino: node.Inode,
NLink: node.Links,
NotReg: node.Type != data.NodeTypeDir && node.Type != data.NodeTypeFile,
NotReg: node.Type != restic.NodeTypeDir && node.Type != restic.NodeTypeFile,
UID: node.UID,
GID: node.GID,
Mode: uint16(node.Mode & os.ModePerm),
@@ -244,13 +240,13 @@ func lsNcduNode(_ string, node *data.Node) ([]byte, error) {
return json.Marshal(outNode)
}
func (p *ncduLsPrinter) Node(path string, node *data.Node, _ bool) error {
func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) error {
out, err := lsNcduNode(path, node)
if err != nil {
return err
}
if node.Type == data.NodeTypeDir {
if node.Type == restic.NodeTypeDir {
_, err = fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out))
p.depth++
} else {
@@ -274,19 +270,15 @@ type textLsPrinter struct {
dirs []string
ListLong bool
HumanReadable bool
termPrinter interface {
P(msg string, args ...interface{})
S(msg string, args ...interface{})
}
}
func (p *textLsPrinter) Snapshot(sn *data.Snapshot) error {
p.termPrinter.P("%v filtered by %v:", sn, p.dirs)
func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) error {
Verbosef("%v filtered by %v:\n", sn, p.dirs)
return nil
}
func (p *textLsPrinter) Node(path string, node *data.Node, isPrefixDirectory bool) error {
func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
if !isPrefixDirectory {
p.termPrinter.S("%s", formatNode(path, node, p.ListLong, p.HumanReadable))
Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
}
return nil
}
@@ -301,12 +293,10 @@ func (p *textLsPrinter) Close() error {
// for ls -l output sorting
type toSortOutput struct {
nodepath string
node *data.Node
node *restic.Node
}
func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []string, term ui.Terminal) error {
termPrinter := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
}
@@ -365,7 +355,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str
return false
}
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, termPrinter)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
@@ -376,7 +366,8 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str
return err
}
if err = repo.LoadIndex(ctx, termPrinter); err != nil {
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}
@@ -384,18 +375,17 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str
if gopts.JSON {
printer = &jsonLsPrinter{
enc: json.NewEncoder(gopts.Term.OutputWriter()),
enc: json.NewEncoder(globalOptions.stdout),
}
} else if opts.Ncdu {
printer = &ncduLsPrinter{
out: gopts.Term.OutputWriter(),
out: globalOptions.stdout,
}
} else {
printer = &textLsPrinter{
dirs: dirs,
ListLong: opts.ListLong,
HumanReadable: opts.HumanReadable,
termPrinter: termPrinter,
}
}
if opts.Sort != SortModeName || opts.Reverse {
@@ -406,12 +396,16 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str
}
}
sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, snapshotLister, repo, args[0])
sn, subfolder, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, snapshotLister, repo, args[0])
if err != nil {
return err
}
sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
@@ -420,7 +414,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str
return err
}
processNode := func(_ restic.ID, nodepath string, node *data.Node, err error) error {
processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
return err
}
@@ -455,7 +449,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts global.Options, args []str
// otherwise, signal the walker to not walk recursively into any
// subdirs
if node.Type == data.NodeTypeDir {
if node.Type == restic.NodeTypeDir {
// immediately generate leaveDir if the directory is skipped
if printedDir {
if err := printer.LeaveDir(nodepath); err != nil {
@@ -492,10 +486,10 @@ type sortedPrinter struct {
reverse bool
}
func (p *sortedPrinter) Snapshot(sn *data.Snapshot) error {
func (p *sortedPrinter) Snapshot(sn *restic.Snapshot) error {
return p.printer.Snapshot(sn)
}
func (p *sortedPrinter) Node(path string, node *data.Node, isPrefixDirectory bool) error {
func (p *sortedPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) error {
if !isPrefixDirectory {
p.collector = append(p.collector, toSortOutput{path, node})
}

View File

@@ -8,22 +8,20 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func testRunLsWithOpts(t testing.TB, gopts global.Options, opts LsOptions, args []string) []byte {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
buf, err := withCaptureStdout(func() error {
gopts.Quiet = true
return runLs(context.TODO(), opts, gopts, args, gopts.Term)
return runLs(context.TODO(), opts, gopts, args)
})
rtest.OK(t, err)
return buf.Bytes()
}
func testRunLs(t testing.TB, gopts global.Options, snapshotID string) []string {
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
return strings.Split(string(out), "\n")
}
@@ -131,7 +129,7 @@ func TestRunLsJson(t *testing.T) {
// partial copy of snapshot structure from cmd_ls
type lsSnapshot struct {
*data.Snapshot
*restic.Snapshot
ID *restic.ID `json:"id"`
ShortID string `json:"short_id"` // deprecated
MessageType string `json:"message_type"` // "snapshot"

View File

@@ -7,13 +7,13 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
type lsTestNode struct {
path string
data.Node
restic.Node
}
var lsTestNodes = []lsTestNode{
@@ -21,9 +21,9 @@ var lsTestNodes = []lsTestNode{
// Permissions, by convention is "-" per mode bit
{
path: "/bar/baz",
Node: data.Node{
Node: restic.Node{
Name: "baz",
Type: data.NodeTypeFile,
Type: restic.NodeTypeFile,
Size: 12345,
UID: 10000000,
GID: 20000000,
@@ -37,9 +37,9 @@ var lsTestNodes = []lsTestNode{
// Even empty files get an explicit size.
{
path: "/foo/empty",
Node: data.Node{
Node: restic.Node{
Name: "empty",
Type: data.NodeTypeFile,
Type: restic.NodeTypeFile,
Size: 0,
UID: 1001,
GID: 1001,
@@ -54,9 +54,9 @@ var lsTestNodes = []lsTestNode{
// Mode is printed in decimal, including the type bits.
{
path: "/foo/link",
Node: data.Node{
Node: restic.Node{
Name: "link",
Type: data.NodeTypeSymlink,
Type: restic.NodeTypeSymlink,
Mode: os.ModeSymlink | 0777,
LinkTarget: "not printed",
},
@@ -64,9 +64,9 @@ var lsTestNodes = []lsTestNode{
{
path: "/some/directory",
Node: data.Node{
Node: restic.Node{
Name: "directory",
Type: data.NodeTypeDir,
Type: restic.NodeTypeDir,
Mode: os.ModeDir | 0755,
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
@@ -77,9 +77,9 @@ var lsTestNodes = []lsTestNode{
// Test encoding of setuid/setgid/sticky bit
{
path: "/some/sticky",
Node: data.Node{
Node: restic.Node{
Name: "sticky",
Type: data.NodeTypeDir,
Type: restic.NodeTypeDir,
Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
},
},
@@ -134,24 +134,24 @@ func TestLsNcdu(t *testing.T) {
}
modTime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
rtest.OK(t, printer.Snapshot(&data.Snapshot{
rtest.OK(t, printer.Snapshot(&restic.Snapshot{
Hostname: "host",
Paths: []string{"/example"},
}))
rtest.OK(t, printer.Node("/directory", &data.Node{
Type: data.NodeTypeDir,
rtest.OK(t, printer.Node("/directory", &restic.Node{
Type: restic.NodeTypeDir,
Name: "directory",
ModTime: modTime,
}, false))
rtest.OK(t, printer.Node("/directory/data", &data.Node{
Type: data.NodeTypeFile,
rtest.OK(t, printer.Node("/directory/data", &restic.Node{
Type: restic.NodeTypeFile,
Name: "data",
Size: 42,
ModTime: modTime,
}, false))
rtest.OK(t, printer.LeaveDir("/directory"))
rtest.OK(t, printer.Node("/file", &data.Node{
Type: data.NodeTypeFile,
rtest.OK(t, printer.Node("/file", &restic.Node{
Type: restic.NodeTypeFile,
Name: "file",
Size: 12345,
ModTime: modTime,

View File

@@ -3,17 +3,16 @@ package main
import (
"context"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/migrations"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newMigrateCommand(globalOptions *global.Options) *cobra.Command {
func newMigrateCommand() *cobra.Command {
var opts MigrateOptions
cmd := &cobra.Command{
@@ -36,7 +35,9 @@ Exit status is 12 if the password is incorrect.
DisableAutoGenTag: true,
GroupID: cmdGroupDefault,
RunE: func(cmd *cobra.Command, args []string) error {
return runMigrate(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
term, cancel := setupTermstatus()
defer cancel()
return runMigrate(cmd.Context(), opts, globalOptions, args, term)
},
}
@@ -76,7 +77,7 @@ func checkMigrations(ctx context.Context, repo restic.Repository, printer progre
return nil
}
func applyMigrations(ctx context.Context, opts MigrateOptions, gopts global.Options, repo restic.Repository, args []string, term ui.Terminal, printer progress.Printer) error {
func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string, term *termstatus.Terminal, printer progress.Printer) error {
var firsterr error
for _, name := range args {
found := false
@@ -134,10 +135,10 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts global.Opti
return firsterr
}
func runMigrate(ctx context.Context, opts MigrateOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error {
printer := newTerminalProgressPrinter(gopts.verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if err != nil {
return err
}

View File

@@ -1,4 +1,5 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package main
@@ -11,13 +12,10 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/sys/unix"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/fuse"
@@ -25,19 +23,19 @@ import (
"github.com/anacrolix/fuse/fs"
)
func registerMountCommand(cmdRoot *cobra.Command, globalOptions *global.Options) {
cmdRoot.AddCommand(newMountCommand(globalOptions))
func registerMountCommand(cmdRoot *cobra.Command) {
cmdRoot.AddCommand(newMountCommand())
}
func newMountCommand(globalOptions *global.Options) *cobra.Command {
func newMountCommand() *cobra.Command {
var opts MountOptions
cmd := &cobra.Command{
Use: "mount [flags] mountpoint",
Short: "Mount the repository",
Long: `
The "mount" command mounts the repository via fuse over a writeable directory.
The repository will be mounted read-only.
The "mount" command mounts the repository via fuse to a directory. This is a
read-only mount.
Snapshot Directories
====================
@@ -83,8 +81,7 @@ Exit status is 12 if the password is incorrect.
DisableAutoGenTag: true,
GroupID: cmdGroupDefault,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runMount(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
return runMount(cmd.Context(), opts, globalOptions, args)
},
}
@@ -97,7 +94,7 @@ type MountOptions struct {
OwnerRoot bool
AllowOther bool
NoDefaultPermissions bool
data.SnapshotFilter
restic.SnapshotFilter
TimeTemplate string
PathTemplates []string
}
@@ -115,9 +112,7 @@ func (opts *MountOptions) AddFlags(f *pflag.FlagSet) {
_ = f.MarkDeprecated("snapshot-template", "use --time-template")
}
func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error {
if opts.TimeTemplate == "" {
return errors.Fatal("time template string cannot be empty")
}
@@ -134,31 +129,22 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
// Check the existence of the mount point at the earliest stage to
// prevent unnecessary computations while opening the repository.
stat, err := os.Stat(mountpoint)
if errors.Is(err, os.ErrNotExist) {
printer.P("Mountpoint %s doesn't exist", mountpoint)
return errors.Fatal("invalid mountpoint")
} else if !stat.IsDir() {
printer.P("Mountpoint %s is not a directory", mountpoint)
return errors.Fatal("invalid mountpoint")
}
err = unix.Access(mountpoint, unix.W_OK|unix.X_OK)
if err != nil {
printer.P("Mountpoint %s is not writeable or not excutable", mountpoint)
return errors.Fatal("inaccessible mountpoint")
if _, err := os.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
return err
}
debug.Log("start mount")
defer debug.Log("finish mount")
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
defer unlock()
err = repo.LoadIndex(ctx, printer)
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
@@ -197,9 +183,9 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
}
root := fuse.NewRoot(repo, cfg)
printer.S("Now serving the repository at %s", mountpoint)
printer.S("Use another terminal or tool to browse the contents of this folder.")
printer.S("When finished, quit with Ctrl-c here or umount the mountpoint.")
Printf("Now serving the repository at %s\n", mountpoint)
Printf("Use another terminal or tool to browse the contents of this folder.\n")
Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
debug.Log("serving mount at %v", mountpoint)
@@ -215,7 +201,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts global.Options, args
debug.Log("running umount cleanup handler for mount at %v", mountpoint)
err := systemFuse.Unmount(mountpoint)
if err != nil {
printer.E("unable to umount (maybe already umounted or still in use?): %v", err)
Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
}
return ErrOK

View File

@@ -1,12 +1,10 @@
//go:build !darwin && !freebsd && !linux
// +build !darwin,!freebsd,!linux
package main
import (
"github.com/restic/restic/internal/global"
"github.com/spf13/cobra"
)
import "github.com/spf13/cobra"
func registerMountCommand(_ *cobra.Command, _ *global.Options) {
func registerMountCommand(_ *cobra.Command) {
// Mount command not supported on these platforms
}

View File

@@ -1,4 +1,5 @@
//go:build darwin || freebsd || linux
// +build darwin freebsd linux
package main
@@ -12,12 +13,9 @@ import (
"time"
systemFuse "github.com/anacrolix/fuse"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
)
const (
@@ -58,14 +56,12 @@ func waitForMount(t testing.TB, dir string) {
t.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir)
}
func testRunMount(t testing.TB, gopts global.Options, dir string, wg *sync.WaitGroup) {
func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGroup) {
defer wg.Done()
opts := MountOptions{
TimeTemplate: time.RFC3339,
}
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runMount(context.TODO(), opts, gopts, []string{dir}, gopts.Term)
}))
rtest.OK(t, runMount(context.TODO(), opts, gopts, []string{dir}))
}
func testRunUmount(t testing.TB, dir string) {
@@ -91,7 +87,7 @@ func listSnapshots(t testing.TB, dir string) []string {
return names
}
func checkSnapshots(t testing.TB, gopts global.Options, mountpoint string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
func checkSnapshots(t testing.TB, gopts GlobalOptions, mountpoint string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
var wg sync.WaitGroup
@@ -129,41 +125,34 @@ func checkSnapshots(t testing.TB, gopts global.Options, mountpoint string, snaps
}
}
err := withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
if err != nil {
return err
_, repo, unlock, err := openWithReadLock(context.TODO(), gopts, false)
rtest.OK(t, err)
defer unlock()
for _, id := range snapshotIDs {
snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
rtest.OK(t, err)
ts := snapshot.Time.Format(time.RFC3339)
present, ok := namesMap[ts]
if !ok {
t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
}
defer unlock()
for _, id := range snapshotIDs {
snapshot, err := data.LoadSnapshot(ctx, repo, id)
rtest.OK(t, err)
ts := snapshot.Time.Format(time.RFC3339)
present, ok := namesMap[ts]
for i := 1; present; i++ {
ts = fmt.Sprintf("%s-%d", snapshot.Time.Format(time.RFC3339), i)
present, ok = namesMap[ts]
if !ok {
t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
}
for i := 1; present; i++ {
ts = fmt.Sprintf("%s-%d", snapshot.Time.Format(time.RFC3339), i)
present, ok = namesMap[ts]
if !ok {
t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
}
if !present {
break
}
if !present {
break
}
namesMap[ts] = true
}
return nil
})
rtest.OK(t, err)
namesMap[ts] = true
}
for name, present := range namesMap {
rtest.Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
@@ -177,7 +166,7 @@ func TestMount(t *testing.T) {
env, cleanup := withTestEnvironment(t)
// must list snapshots more than once
env.gopts.BackendTestHook = nil
env.gopts.backendTestHook = nil
defer cleanup()
testRunInit(t, env.gopts)
@@ -188,7 +177,7 @@ func TestMount(t *testing.T) {
// first backup
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
snapshotIDs := testRunList(t, env.gopts, "snapshots")
snapshotIDs := testRunList(t, "snapshots", env.gopts)
rtest.Assert(t, len(snapshotIDs) == 1,
"expected one snapshot, got %v", snapshotIDs)
@@ -196,7 +185,7 @@ func TestMount(t *testing.T) {
// second backup, implicit incremental
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
snapshotIDs = testRunList(t, env.gopts, "snapshots")
snapshotIDs = testRunList(t, "snapshots", env.gopts)
rtest.Assert(t, len(snapshotIDs) == 2,
"expected two snapshots, got %v", snapshotIDs)
@@ -205,7 +194,7 @@ func TestMount(t *testing.T) {
// third backup, explicit incremental
bopts := BackupOptions{Parent: snapshotIDs[0].String()}
testRunBackup(t, "", []string{env.testdata}, bopts, env.gopts)
snapshotIDs = testRunList(t, env.gopts, "snapshots")
snapshotIDs = testRunList(t, "snapshots", env.gopts)
rtest.Assert(t, len(snapshotIDs) == 3,
"expected three snapshots, got %v", snapshotIDs)
@@ -224,7 +213,7 @@ func TestMountSameTimestamps(t *testing.T) {
env, cleanup := withTestEnvironment(t)
// must list snapshots more than once
env.gopts.BackendTestHook = nil
env.gopts.backendTestHook = nil
defer cleanup()
rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))

View File

@@ -3,13 +3,12 @@ package main
import (
"fmt"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/options"
"github.com/spf13/cobra"
)
func newOptionsCommand(globalOptions *global.Options) *cobra.Command {
func newOptionsCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "options",
Short: "Print list of extended options",
@@ -25,7 +24,7 @@ Exit status is 1 if there was any error.
GroupID: cmdGroupAdvanced,
DisableAutoGenTag: true,
Run: func(_ *cobra.Command, _ []string) {
globalOptions.Term.Print("All Extended Options:")
fmt.Printf("All Extended Options:\n")
var maxLen int
for _, opt := range options.List() {
if l := len(opt.Namespace + "." + opt.Name); l > maxLen {
@@ -33,7 +32,7 @@ Exit status is 1 if there was any error.
}
}
for _, opt := range options.List() {
globalOptions.Term.Print(fmt.Sprintf(" %*s %s", -maxLen, opt.Namespace+"."+opt.Name, opt.Text))
fmt.Printf(" %*s %s\n", -maxLen, opt.Namespace+"."+opt.Name, opt.Text)
}
},
}

View File

@@ -7,20 +7,19 @@ import (
"strconv"
"strings"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newPruneCommand(globalOptions *global.Options) *cobra.Command {
func newPruneCommand() *cobra.Command {
var opts PruneOptions
cmd := &cobra.Command{
@@ -42,7 +41,9 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
return runPrune(cmd.Context(), opts, *globalOptions, globalOptions.Term)
term, cancel := setupTermstatus()
defer cancel()
return runPrune(cmd.Context(), opts, globalOptions, term)
},
}
@@ -154,7 +155,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
return nil
}
func runPrune(ctx context.Context, opts PruneOptions, gopts global.Options, term ui.Terminal) error {
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
err := verifyPruneOptions(&opts)
if err != nil {
return err
@@ -168,8 +169,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts global.Options, term
return errors.Fatal("--no-lock is only applicable in combination with --dry-run for prune command")
}
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
if err != nil {
return err
}
@@ -183,16 +183,20 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts global.Options, term
opts.unsafeRecovery = true
}
return runPruneWithRepo(ctx, opts, repo, restic.NewIDSet(), printer)
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), term)
}
func runPruneWithRepo(ctx context.Context, opts PruneOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) error {
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error {
if repo.Cache() == nil {
printer.S("warning: running prune without a cache, this may be very slow!")
Print("warning: running prune without a cache, this may be very slow!\n")
}
printer := newTerminalProgressPrinter(gopts.verbosity, term)
printer.P("loading indexes...\n")
// loading the index before the snapshots is ok, as we use an exclusive lock here
err := repo.LoadIndex(ctx, printer)
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err := repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
@@ -280,8 +284,8 @@ func printPruneStats(printer progress.Printer, stats repository.PruneStats) erro
func getUsedBlobs(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet, ignoreSnapshots restic.IDSet, printer progress.Printer) error {
var snapshotTrees restic.IDs
printer.P("loading all snapshots...\n")
err := data.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
func(id restic.ID, sn *data.Snapshot, err error) error {
err := restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
debug.Log("failed to load snapshot %v (error %v)", id, err)
return err
@@ -300,10 +304,5 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, usedBlobs restic.
bar.SetMax(uint64(len(snapshotTrees)))
defer bar.Done()
err = data.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
if err != nil {
return errors.Fatalf("failed finding blobs: %v", err)
}
return nil
return restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
}

View File

@@ -7,30 +7,30 @@ import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunPrune(t testing.TB, gopts global.Options, opts PruneOptions) {
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
t.Helper()
rtest.OK(t, testRunPruneOutput(t, gopts, opts))
rtest.OK(t, testRunPruneOutput(gopts, opts))
}
func testRunPruneMustFail(t testing.TB, gopts global.Options, opts PruneOptions) {
func testRunPruneMustFail(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
t.Helper()
err := testRunPruneOutput(t, gopts, opts)
err := testRunPruneOutput(gopts, opts)
rtest.Assert(t, err != nil, "expected non nil error")
}
func testRunPruneOutput(t testing.TB, gopts global.Options, opts PruneOptions) error {
oldHook := gopts.BackendTestHook
gopts.BackendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
func testRunPruneOutput(gopts GlobalOptions, opts PruneOptions) error {
oldHook := gopts.backendTestHook
gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() {
gopts.BackendTestHook = oldHook
gopts.backendTestHook = oldHook
}()
return withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runPrune(context.TODO(), opts, gopts, gopts.Term)
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runPrune(context.TODO(), opts, gopts, term)
})
}
@@ -89,8 +89,8 @@ func createPrunableRepo(t *testing.T, env *testEnvironment) {
testRunForget(t, env.gopts, ForgetOptions{}, firstSnapshot.String())
}
func testRunForgetJSON(t testing.TB, gopts global.Options, args ...string) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
buf, err := withCaptureStdout(func() error {
gopts.JSON = true
opts := ForgetOptions{
DryRun: true,
@@ -99,7 +99,9 @@ func testRunForgetJSON(t testing.TB, gopts global.Options, args ...string) {
pruneOpts := PruneOptions{
MaxUnused: "5%",
}
return runForget(context.TODO(), opts, pruneOpts, gopts, gopts.Term, args)
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
})
})
rtest.OK(t, err)
@@ -120,8 +122,8 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
createPrunableRepo(t, env)
testRunPrune(t, env.gopts, pruneOpts)
rtest.OK(t, withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
_, err := runCheck(context.TODO(), checkOpts, gopts, nil, gopts.Term)
rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
_, err := runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
return err
}))
}
@@ -150,14 +152,14 @@ func TestPruneWithDamagedRepository(t *testing.T) {
testListSnapshots(t, env.gopts, 1)
removePacksExcept(env.gopts, t, oldPacks, false)
oldHook := env.gopts.BackendTestHook
env.gopts.BackendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
oldHook := env.gopts.backendTestHook
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
defer func() {
env.gopts.BackendTestHook = oldHook
env.gopts.backendTestHook = oldHook
}()
// prune should fail
rtest.Equals(t, repository.ErrPacksMissing, withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runPrune(context.TODO(), pruneDefaultOptions, gopts, gopts.Term)
rtest.Equals(t, repository.ErrPacksMissing, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term)
}), "prune should have reported index not complete error")
}
@@ -229,8 +231,8 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
if checkOK {
testRunCheck(t, env.gopts)
} else {
rtest.Assert(t, withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
_, err := runCheck(context.TODO(), optionsCheck, gopts, nil, gopts.Term)
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
_, err := runCheck(context.TODO(), optionsCheck, env.gopts, nil, term)
return err
}) != nil,
"check should have reported an error")
@@ -240,8 +242,8 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
testRunPrune(t, env.gopts, optionsPrune)
testRunCheck(t, env.gopts)
} else {
rtest.Assert(t, withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runPrune(context.TODO(), optionsPrune, gopts, gopts.Term)
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runPrune(context.TODO(), optionsPrune, env.gopts, term)
}) != nil,
"prune should have reported an error")
}

View File

@@ -5,17 +5,16 @@ import (
"os"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
)
func newRecoverCommand(globalOptions *global.Options) *cobra.Command {
func newRecoverCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "recover [flags]",
Short: "Recover data from the repository not referenced by snapshots",
@@ -36,25 +35,28 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
return runRecover(cmd.Context(), *globalOptions, globalOptions.Term)
term, cancel := setupTermstatus()
defer cancel()
return runRecover(cmd.Context(), globalOptions, term)
},
}
return cmd
}
func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) error {
func runRecover(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal) error {
hostname, err := os.Hostname()
if err != nil {
return err
}
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if err != nil {
return err
}
defer unlock()
printer := newTerminalProgressPrinter(gopts.verbosity, term)
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
if err != nil {
return err
@@ -67,7 +69,8 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
}
printer.P("load index files\n")
if err = repo.LoadIndex(ctx, printer); err != nil {
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
if err = repo.LoadIndex(ctx, bar); err != nil {
return err
}
@@ -85,10 +88,9 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
}
printer.P("load %d trees\n", len(trees))
bar := printer.NewCounter("trees loaded")
bar.SetMax(uint64(len(trees)))
bar = newTerminalProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded", term)
for id := range trees {
tree, err := data.LoadTree(ctx, repo, id)
tree, err := restic.LoadTree(ctx, repo, id)
if ctx.Err() != nil {
return ctx.Err()
}
@@ -97,12 +99,8 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
continue
}
for item := range tree {
if item.Error != nil {
return item.Error
}
node := item.Node
if node.Type == data.NodeTypeDir && node.Subtree != nil {
for _, node := range tree.Nodes {
if node.Type == restic.NodeTypeDir && node.Subtree != nil {
trees[*node.Subtree] = true
}
}
@@ -111,7 +109,7 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
bar.Done()
printer.P("load snapshots\n")
err = data.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *data.Snapshot, _ error) error {
err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error {
trees[*sn.Tree] = true
return nil
})
@@ -138,33 +136,42 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
return ctx.Err()
}
var treeID restic.ID
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
var err error
tw := data.NewTreeWriter(uploader)
for id := range roots {
var subtreeID = id
node := data.Node{
Type: data.NodeTypeDir,
Name: id.Str(),
Mode: 0755,
Subtree: &subtreeID,
AccessTime: time.Now(),
ModTime: time.Now(),
ChangeTime: time.Now(),
}
err := tw.AddNode(&node)
if err != nil {
return err
}
tree := restic.NewTree(len(roots))
for id := range roots {
var subtreeID = id
node := restic.Node{
Type: restic.NodeTypeDir,
Name: id.Str(),
Mode: 0755,
Subtree: &subtreeID,
AccessTime: time.Now(),
ModTime: time.Now(),
ChangeTime: time.Now(),
}
err := tree.Insert(&node)
if err != nil {
return err
}
}
treeID, err = tw.Finalize(ctx)
wg, wgCtx := errgroup.WithContext(ctx)
repo.StartPackUploader(wgCtx, wg)
var treeID restic.ID
wg.Go(func() error {
var err error
treeID, err = restic.SaveTree(wgCtx, repo, tree)
if err != nil {
return errors.Fatalf("unable to save new tree to the repository: %v", err)
}
err = repo.Flush(wgCtx)
if err != nil {
return errors.Fatalf("unable to save blobs to the repository: %v", err)
}
return nil
})
err = wg.Wait()
if err != nil {
return err
}
@@ -174,14 +181,14 @@ func runRecover(ctx context.Context, gopts global.Options, term ui.Terminal) err
}
func createSnapshot(ctx context.Context, printer progress.Printer, name, hostname string, tags []string, repo restic.SaverUnpacked[restic.WriteableFileType], tree *restic.ID) error {
sn, err := data.NewSnapshot([]string{name}, tags, hostname, time.Now())
sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
if err != nil {
return errors.Fatalf("unable to save snapshot: %v", err)
}
sn.Tree = tree
id, err := data.SaveSnapshot(ctx, repo, sn)
id, err := restic.SaveSnapshot(ctx, repo, sn)
if err != nil {
return errors.Fatalf("unable to save snapshot: %v", err)
}

View File

@@ -4,20 +4,20 @@ import (
"context"
"testing"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunRecover(t testing.TB, gopts global.Options) {
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runRecover(context.TODO(), gopts, gopts.Term)
func testRunRecover(t testing.TB, gopts GlobalOptions) {
rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runRecover(context.TODO(), gopts, term)
}))
}
func TestRecover(t *testing.T) {
env, cleanup := withTestEnvironment(t)
// must list index more than once
env.gopts.BackendTestHook = nil
env.gopts.backendTestHook = nil
defer cleanup()
testSetupBackupData(t, env)
@@ -33,7 +33,5 @@ func TestRecover(t *testing.T) {
ids = testListSnapshots(t, env.gopts, 1)
testRunCheck(t, env.gopts)
// check that the root tree is included in the snapshot
rtest.OK(t, withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
return runCat(context.TODO(), gopts, []string{"tree", ids[0].String() + ":" + sn.Tree.Str()}, gopts.Term)
}))
rtest.OK(t, runCat(context.TODO(), env.gopts, []string{"tree", ids[0].String() + ":" + sn.Tree.Str()}))
}

View File

@@ -1,11 +1,10 @@
package main
import (
"github.com/restic/restic/internal/global"
"github.com/spf13/cobra"
)
func newRepairCommand(globalOptions *global.Options) *cobra.Command {
func newRepairCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "repair",
Short: "Repair the repository",
@@ -14,9 +13,9 @@ func newRepairCommand(globalOptions *global.Options) *cobra.Command {
}
cmd.AddCommand(
newRepairIndexCommand(globalOptions),
newRepairPacksCommand(globalOptions),
newRepairSnapshotsCommand(globalOptions),
newRepairIndexCommand(),
newRepairPacksCommand(),
newRepairSnapshotsCommand(),
)
return cmd
}

View File

@@ -3,14 +3,13 @@ package main
import (
"context"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newRepairIndexCommand(globalOptions *global.Options) *cobra.Command {
func newRepairIndexCommand() *cobra.Command {
var opts RepairIndexOptions
cmd := &cobra.Command{
@@ -31,7 +30,9 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
return runRebuildIndex(cmd.Context(), opts, *globalOptions, globalOptions.Term)
term, cancel := setupTermstatus()
defer cancel()
return runRebuildIndex(cmd.Context(), opts, globalOptions, term)
},
}
@@ -48,10 +49,10 @@ func (opts *RepairIndexOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
}
func newRebuildIndexCommand(globalOptions *global.Options) *cobra.Command {
func newRebuildIndexCommand() *cobra.Command {
var opts RepairIndexOptions
replacement := newRepairIndexCommand(globalOptions)
replacement := newRepairIndexCommand()
cmd := &cobra.Command{
Use: "rebuild-index [flags]",
Short: replacement.Short,
@@ -61,7 +62,9 @@ func newRebuildIndexCommand(globalOptions *global.Options) *cobra.Command {
// must create a new instance of the run function as it captures opts
// by reference
RunE: func(cmd *cobra.Command, _ []string) error {
return runRebuildIndex(cmd.Context(), opts, *globalOptions, globalOptions.Term)
term, cancel := setupTermstatus()
defer cancel()
return runRebuildIndex(cmd.Context(), opts, globalOptions, term)
},
}
@@ -69,15 +72,15 @@ func newRebuildIndexCommand(globalOptions *global.Options) *cobra.Command {
return cmd
}
func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts global.Options, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false, printer)
func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if err != nil {
return err
}
defer unlock()
printer := newTerminalProgressPrinter(gopts.verbosity, term)
err = repository.RepairIndex(ctx, repo, repository.RepairIndexOptions{
ReadAllPacks: opts.ReadAllPacks,
}, printer)

View File

@@ -10,27 +10,29 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunRebuildIndex(t testing.TB, gopts global.Options) {
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
gopts.Quiet = true
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, gopts.Term)
func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
rtest.OK(t, withRestoreGlobalOptions(func() error {
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
globalOptions.stdout = io.Discard
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, term)
})
}))
}
func testRebuildIndex(t *testing.T, backendTestHook global.BackendWrapper) {
func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
datafile := filepath.Join("..", "..", "internal", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
rtest.SetupTarTestFixture(t, env.base, datafile)
out, err := testRunCheckOutput(t, env.gopts, false)
out, err := testRunCheckOutput(env.gopts, false)
if !strings.Contains(out, "contained in several indexes") {
t.Fatalf("did not find checker hint for packs in several indexes")
}
@@ -43,11 +45,11 @@ func testRebuildIndex(t *testing.T, backendTestHook global.BackendWrapper) {
t.Fatalf("did not find hint for repair index command")
}
env.gopts.BackendTestHook = backendTestHook
env.gopts.backendTestHook = backendTestHook
testRunRebuildIndex(t, env.gopts)
env.gopts.BackendTestHook = nil
out, err = testRunCheckOutput(t, env.gopts, false)
env.gopts.backendTestHook = nil
out, err = testRunCheckOutput(env.gopts, false)
if len(out) != 0 {
t.Fatalf("expected no output from the checker, got: %v", out)
}
@@ -126,12 +128,14 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
datafile := filepath.Join("..", "..", "internal", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
rtest.SetupTarTestFixture(t, env.base, datafile)
env.gopts.BackendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &appendOnlyBackend{r}, nil
}
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
gopts.Quiet = true
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, gopts.Term)
err := withRestoreGlobalOptions(func() error {
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
return &appendOnlyBackend{r}, nil
}
return withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
globalOptions.stdout = io.Discard
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
})
})
if err == nil {

View File

@@ -7,14 +7,13 @@ import (
"os"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
)
func newRepairPacksCommand(globalOptions *global.Options) *cobra.Command {
func newRepairPacksCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "packs [packIDs...]",
Short: "Salvage damaged pack files",
@@ -33,13 +32,15 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runRepairPacks(cmd.Context(), *globalOptions, globalOptions.Term, args)
term, cancel := setupTermstatus()
defer cancel()
return runRepairPacks(cmd.Context(), globalOptions, term, args)
},
}
return cmd
}
func runRepairPacks(ctx context.Context, gopts global.Options, term ui.Terminal, args []string) error {
func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
ids := restic.NewIDSet()
for _, arg := range args {
id, err := restic.ParseID(arg)
@@ -52,15 +53,16 @@ func runRepairPacks(ctx context.Context, gopts global.Options, term ui.Terminal,
return errors.Fatal("no ids specified")
}
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false, printer)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
if err != nil {
return err
}
defer unlock()
err = repo.LoadIndex(ctx, printer)
printer := newTerminalProgressPrinter(gopts.verbosity, term)
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return errors.Fatalf("%s", err)
}
@@ -91,6 +93,6 @@ func runRepairPacks(ctx context.Context, gopts global.Options, term ui.Terminal,
return errors.Fatalf("%s", err)
}
printer.E("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots")
Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n")
return nil
}

View File

@@ -2,20 +2,16 @@ package main
import (
"context"
"slices"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/walker"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newRepairSnapshotsCommand(globalOptions *global.Options) *cobra.Command {
func newRepairSnapshotsCommand() *cobra.Command {
var opts RepairOptions
cmd := &cobra.Command{
@@ -53,8 +49,7 @@ Exit status is 12 if the password is incorrect.
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runRepairSnapshots(cmd.Context(), *globalOptions, opts, args, globalOptions.Term)
return runRepairSnapshots(cmd.Context(), globalOptions, opts, args)
},
}
@@ -67,7 +62,7 @@ type RepairOptions struct {
DryRun bool
Forget bool
data.SnapshotFilter
restic.SnapshotFilter
}
func (opts *RepairOptions) AddFlags(f *pflag.FlagSet) {
@@ -77,10 +72,8 @@ func (opts *RepairOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
}
func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOptions, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun, printer)
func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error {
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun)
if err != nil {
return err
}
@@ -91,7 +84,8 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
return err
}
if err := repo.LoadIndex(ctx, printer); err != nil {
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err := repo.LoadIndex(ctx, bar); err != nil {
return err
}
@@ -100,12 +94,12 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
// - trees which cannot be loaded (-> the tree contents will be removed)
// - files whose contents are not fully available (-> file will be modified)
rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
RewriteNode: func(node *data.Node, path string) *data.Node {
if node.Type == data.NodeTypeIrregular || node.Type == data.NodeTypeInvalid {
printer.P(" file %q: removed node with invalid type %q", path, node.Type)
RewriteNode: func(node *restic.Node, path string) *restic.Node {
if node.Type == restic.NodeTypeIrregular || node.Type == restic.NodeTypeInvalid {
Verbosef(" file %q: removed node with invalid type %q\n", path, node.Type)
return nil
}
if node.Type != data.NodeTypeFile {
if node.Type != restic.NodeTypeFile {
return node
}
@@ -122,36 +116,40 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
}
}
if !ok {
printer.P(" file %q: removed missing content", path)
Verbosef(" file %q: removed missing content\n", path)
} else if newSize != node.Size {
printer.P(" file %q: fixed incorrect size", path)
Verbosef(" file %q: fixed incorrect size\n", path)
}
// no-ops if already correct
node.Content = newContent
node.Size = newSize
return node
},
RewriteFailedTree: func(_ restic.ID, path string, _ error) (data.TreeNodeIterator, error) {
RewriteFailedTree: func(_ restic.ID, path string, _ error) (restic.ID, error) {
if path == "/" {
printer.P(" dir %q: not readable", path)
Verbosef(" dir %q: not readable\n", path)
// remove snapshots with invalid root node
return nil, nil
return restic.ID{}, nil
}
// If a subtree fails to load, remove it
printer.P(" dir %q: replaced with empty directory", path)
return slices.Values([]data.NodeOrError{}), nil
Verbosef(" dir %q: replaced with empty directory\n", path)
emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{})
if err != nil {
return restic.ID{}, err
}
return emptyID, nil
},
AllowUnstableSerialization: true,
})
changedCount := 0
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args, printer) {
printer.P("\n%v", sn)
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
Verbosef("\n%v\n", sn)
changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
func(ctx context.Context, sn *data.Snapshot, uploader restic.BlobSaver) (restic.ID, *data.SnapshotSummary, error) {
id, err := rewriter.RewriteTree(ctx, repo, uploader, "/", *sn.Tree)
func(ctx context.Context, sn *restic.Snapshot) (restic.ID, *restic.SnapshotSummary, error) {
id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
return id, nil, err
}, opts.DryRun, opts.Forget, nil, "repaired", printer, false)
}, opts.DryRun, opts.Forget, nil, "repaired")
if err != nil {
return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
}
@@ -163,18 +161,18 @@ func runRepairSnapshots(ctx context.Context, gopts global.Options, opts RepairOp
return ctx.Err()
}
printer.P("")
Verbosef("\n")
if changedCount == 0 {
if !opts.DryRun {
printer.P("no snapshots were modified")
Verbosef("no snapshots were modified\n")
} else {
printer.P("no snapshots would be modified")
Verbosef("no snapshots would be modified\n")
}
} else {
if !opts.DryRun {
printer.P("modified %v snapshots", changedCount)
Verbosef("modified %v snapshots\n", changedCount)
} else {
printer.P("would modify %v snapshots", changedCount)
Verbosef("would modify %v snapshots\n", changedCount)
}
}

View File

@@ -10,19 +10,16 @@ import (
"reflect"
"testing"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func testRunRepairSnapshot(t testing.TB, gopts global.Options, forget bool) {
func testRunRepairSnapshot(t testing.TB, gopts GlobalOptions, forget bool) {
opts := RepairOptions{
Forget: forget,
}
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runRepairSnapshots(context.TODO(), gopts, opts, nil, gopts.Term)
}))
rtest.OK(t, runRepairSnapshots(context.TODO(), gopts, opts, nil))
}
func createRandomFile(t testing.TB, env *testEnvironment, path string, size int) {
@@ -67,7 +64,7 @@ func TestRepairSnapshotsWithLostData(t *testing.T) {
// repository must be ok after removing the broken snapshots
testRunForget(t, env.gopts, ForgetOptions{}, snapshotIDs[0].String(), snapshotIDs[1].String())
testListSnapshots(t, env.gopts, 2)
_, err := testRunCheckOutput(t, env.gopts, false)
_, err := testRunCheckOutput(env.gopts, false)
rtest.OK(t, err)
}
@@ -80,7 +77,7 @@ func TestRepairSnapshotsWithLostTree(t *testing.T) {
createRandomFile(t, env, "foo/bar/file", 12345)
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
oldSnapshot := testListSnapshots(t, env.gopts, 1)
oldPacks := testRunList(t, env.gopts, "packs")
oldPacks := testRunList(t, "packs", env.gopts)
// keep foo/bar unchanged
createRandomFile(t, env, "foo/bar2", 1024)
@@ -96,7 +93,7 @@ func TestRepairSnapshotsWithLostTree(t *testing.T) {
testRunRebuildIndex(t, env.gopts)
testRunRepairSnapshot(t, env.gopts, true)
testListSnapshots(t, env.gopts, 1)
_, err := testRunCheckOutput(t, env.gopts, false)
_, err := testRunCheckOutput(env.gopts, false)
rtest.OK(t, err)
}
@@ -109,7 +106,7 @@ func TestRepairSnapshotsWithLostRootTree(t *testing.T) {
createRandomFile(t, env, "foo/bar/file", 12345)
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
testListSnapshots(t, env.gopts, 1)
oldPacks := testRunList(t, env.gopts, "packs")
oldPacks := testRunList(t, "packs", env.gopts)
// remove all trees
removePacks(env.gopts, t, restic.NewIDSet(oldPacks...))
@@ -119,7 +116,7 @@ func TestRepairSnapshotsWithLostRootTree(t *testing.T) {
testRunRebuildIndex(t, env.gopts)
testRunRepairSnapshot(t, env.gopts, true)
testListSnapshots(t, env.gopts, 0)
_, err := testRunCheckOutput(t, env.gopts, false)
_, err := testRunCheckOutput(env.gopts, false)
rtest.OK(t, err)
}

View File

@@ -3,24 +3,22 @@ package main
import (
"context"
"path/filepath"
"runtime"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/restorer"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
restoreui "github.com/restic/restic/internal/ui/restore"
"github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func newRestoreCommand(globalOptions *global.Options) *cobra.Command {
func newRestoreCommand() *cobra.Command {
var opts RestoreOptions
cmd := &cobra.Command{
@@ -36,8 +34,6 @@ repository.
To only restore a specific subfolder, you can use the "snapshotID:subfolder"
syntax, where "subfolder" is a path within the snapshot.
POSIX ACLs are always restored by their numeric value, while file ownership can optionally be restored by name instead of numeric value.
EXIT STATUS
===========
@@ -50,8 +46,9 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter)
return runRestore(cmd.Context(), opts, *globalOptions, globalOptions.Term, args)
term, cancel := setupTermstatus()
defer cancel()
return runRestore(cmd.Context(), opts, globalOptions, term, args)
},
}
@@ -64,7 +61,7 @@ type RestoreOptions struct {
filter.ExcludePatternOptions
filter.IncludePatternOptions
Target string
data.SnapshotFilter
restic.SnapshotFilter
DryRun bool
Sparse bool
Verify bool
@@ -72,7 +69,6 @@ type RestoreOptions struct {
Delete bool
ExcludeXattrPattern []string
IncludeXattrPattern []string
OwnershipByName bool
}
func (opts *RestoreOptions) AddFlags(f *pflag.FlagSet) {
@@ -90,27 +86,17 @@ func (opts *RestoreOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.Verify, "verify", false, "verify restored files content")
f.Var(&opts.Overwrite, "overwrite", "overwrite behavior, one of (always|if-changed|if-newer|never)")
f.BoolVar(&opts.Delete, "delete", false, "delete files from target directory if they do not exist in snapshot. Use '--dry-run -vv' to check what would be deleted")
if runtime.GOOS != "windows" {
f.BoolVar(&opts.OwnershipByName, "ownership-by-name", false, "restore file ownership by user name and group name (except POSIX ACLs)")
}
}
func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
term ui.Terminal, args []string) error {
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
term *termstatus.Terminal, args []string) error {
var printer restoreui.ProgressPrinter
if gopts.JSON {
printer = restoreui.NewJSONProgress(term, gopts.Verbosity)
} else {
printer = restoreui.NewTextProgress(term, gopts.Verbosity)
}
excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(printer.E)
excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
if err != nil {
return err
}
includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(printer.E)
includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(Warnf)
if err != nil {
return err
}
@@ -145,35 +131,47 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
debug.Log("restore %v to %v", snapshotIDString, opts.Target)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil {
return err
}
defer unlock()
sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString)
sn, subfolder, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err)
}
err = repo.LoadIndex(ctx, printer)
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err = repo.LoadIndex(ctx, bar)
if err != nil {
return err
}
sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil {
return err
}
progress := restoreui.NewProgress(printer, ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus()))
msg := ui.NewMessage(term, gopts.verbosity)
var printer restoreui.ProgressPrinter
if gopts.JSON {
printer = restoreui.NewJSONProgress(term, gopts.verbosity)
} else {
printer = restoreui.NewTextProgress(term, gopts.verbosity)
}
progress := restoreui.NewProgress(printer, calculateProgressInterval(!gopts.Quiet, gopts.JSON))
res := restorer.NewRestorer(repo, sn, restorer.Options{
DryRun: opts.DryRun,
Sparse: opts.Sparse,
Progress: progress,
Overwrite: opts.Overwrite,
Delete: opts.Delete,
OwnershipByName: opts.OwnershipByName,
DryRun: opts.DryRun,
Sparse: opts.Sparse,
Progress: progress,
Overwrite: opts.Overwrite,
Delete: opts.Delete,
})
totalErrors := 0
@@ -182,13 +180,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
return progress.Error(location, err)
}
res.Warn = func(message string) {
printer.E("Warning: %s\n", message)
msg.E("Warning: %s\n", message)
}
res.Info = func(message string) {
if gopts.JSON {
return
}
printer.P("Info: %s\n", message)
msg.P("Info: %s\n", message)
}
selectExcludeFilter := func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) {
@@ -236,13 +234,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
res.SelectFilter = selectIncludeFilter
}
res.XattrSelectFilter, err = getXattrSelectFilter(opts, printer)
res.XattrSelectFilter, err = getXattrSelectFilter(opts)
if err != nil {
return err
}
if !gopts.JSON {
printer.P("restoring %s to %s\n", res.Snapshot(), opts.Target)
msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target)
}
countRestoredFiles, err := res.RestoreTo(ctx, opts.Target)
@@ -253,26 +251,26 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
progress.Finish()
if totalErrors > 0 {
return errors.Fatalf("There were %d errors", totalErrors)
return errors.Fatalf("There were %d errors\n", totalErrors)
}
if opts.Verify {
if !gopts.JSON {
printer.P("verifying files in %s\n", opts.Target)
msg.P("verifying files in %s\n", opts.Target)
}
var count int
t0 := time.Now()
bar := printer.NewCounterTerminalOnly("files verified")
bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON && stdoutIsTerminal(), 0, "files verified", term)
count, err = res.VerifyFiles(ctx, opts.Target, countRestoredFiles, bar)
if err != nil {
return err
}
if totalErrors > 0 {
return errors.Fatalf("There were %d errors", totalErrors)
return errors.Fatalf("There were %d errors\n", totalErrors)
}
if !gopts.JSON {
printer.P("finished verifying %d files in %s (took %s)\n", count, opts.Target,
msg.P("finished verifying %d files in %s (took %s)\n", count, opts.Target,
time.Since(t0).Round(time.Millisecond))
}
}
@@ -280,7 +278,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts global.Options,
return nil
}
func getXattrSelectFilter(opts RestoreOptions, printer progress.Printer) (func(xattrName string) bool, error) {
func getXattrSelectFilter(opts RestoreOptions) (func(xattrName string) bool, error) {
hasXattrExcludes := len(opts.ExcludeXattrPattern) > 0
hasXattrIncludes := len(opts.IncludeXattrPattern) > 0
@@ -294,7 +292,7 @@ func getXattrSelectFilter(opts RestoreOptions, printer progress.Printer) (func(x
}
return func(xattrName string) bool {
shouldReject := filter.RejectByPattern(opts.ExcludeXattrPattern, printer.E)(xattrName)
shouldReject := filter.RejectByPattern(opts.ExcludeXattrPattern, Warnf)(xattrName)
return !shouldReject
}, nil
}
@@ -306,7 +304,7 @@ func getXattrSelectFilter(opts RestoreOptions, printer progress.Printer) (func(x
}
return func(xattrName string) bool {
shouldInclude, _ := filter.IncludeByPattern(opts.IncludeXattrPattern, printer.E)(xattrName)
shouldInclude, _ := filter.IncludeByPattern(opts.IncludeXattrPattern, Warnf)(xattrName)
return shouldInclude
}, nil
}

View File

@@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"io"
"math/rand"
"os"
"path/filepath"
@@ -11,68 +12,67 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
)
func testRunRestore(t testing.TB, gopts global.Options, dir string, snapshotID string) {
testRunRestoreExcludes(t, gopts, dir, snapshotID, nil)
func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID string) {
testRunRestoreExcludes(t, opts, dir, snapshotID, nil)
}
func testRunRestoreExcludes(t testing.TB, gopts global.Options, dir string, snapshotID string, excludes []string) {
func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID string, excludes []string) {
opts := RestoreOptions{
Target: dir,
}
opts.Excludes = excludes
rtest.OK(t, testRunRestoreAssumeFailure(t, snapshotID, opts, gopts))
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID, opts, gopts))
}
func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts global.Options) error {
return withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
return runRestore(ctx, opts, gopts, gopts.Term, []string{snapshotID})
func testRunRestoreAssumeFailure(snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runRestore(ctx, opts, gopts, term, []string{snapshotID})
})
}
func testRunRestoreLatest(t testing.TB, gopts global.Options, dir string, paths []string, hosts []string) {
func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) {
opts := RestoreOptions{
Target: dir,
SnapshotFilter: data.SnapshotFilter{
SnapshotFilter: restic.SnapshotFilter{
Hosts: hosts,
Paths: paths,
},
}
rtest.OK(t, testRunRestoreAssumeFailure(t, "latest", opts, gopts))
rtest.OK(t, testRunRestoreAssumeFailure("latest", opts, gopts))
}
func testRunRestoreIncludes(t testing.TB, gopts global.Options, dir string, snapshotID restic.ID, includes []string) {
func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
opts := RestoreOptions{
Target: dir,
}
opts.Includes = includes
rtest.OK(t, testRunRestoreAssumeFailure(t, snapshotID.String(), opts, gopts))
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func testRunRestoreIncludesFromFile(t testing.TB, gopts global.Options, dir string, snapshotID restic.ID, includesFile string) {
func testRunRestoreIncludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includesFile string) {
opts := RestoreOptions{
Target: dir,
}
opts.IncludeFiles = []string{includesFile}
rtest.OK(t, testRunRestoreAssumeFailure(t, snapshotID.String(), opts, gopts))
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func testRunRestoreExcludesFromFile(t testing.TB, gopts global.Options, dir string, snapshotID restic.ID, excludesFile string) {
func testRunRestoreExcludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludesFile string) {
opts := RestoreOptions{
Target: dir,
}
opts.ExcludeFiles = []string{excludesFile}
rtest.OK(t, testRunRestoreAssumeFailure(t, snapshotID.String(), opts, gopts))
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}
func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) {
@@ -93,7 +93,7 @@ func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) {
restoreOpts.Includes = includePatterns
restoreOpts.Excludes = excludePatterns
err := testRunRestoreAssumeFailure(t, "latest", restoreOpts, env.gopts)
err := testRunRestoreAssumeFailure("latest", restoreOpts, env.gopts)
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "exclude and include patterns are mutually exclusive"),
"expected: %s error, got %v", "exclude and include patterns are mutually exclusive", err)
}
@@ -257,7 +257,7 @@ func TestRestore(t *testing.T) {
restoredir := filepath.Join(env.base, "restore")
testRunRestoreLatest(t, env.gopts, restoredir, nil, nil)
diff := directoriesContentsDiff(t, env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata)))
diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata)))
rtest.Assert(t, diff == "", "directories are not equal %v", diff)
}
@@ -337,7 +337,11 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
snapshots := testListSnapshots(t, env.gopts, 1)
testRunRestore(t, env.gopts, filepath.Join(env.base, "restore"), snapshots[0].String())
_ = withRestoreGlobalOptions(func() error {
globalOptions.stderr = io.Discard
testRunRestore(t, env.gopts, filepath.Join(env.base, "restore"), snapshots[0].String())
return nil
})
// make sure that all files have been restored, regardless of any
// permission errors
@@ -394,7 +398,7 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
fi, err := os.Stat(f2)
rtest.OK(t, err)
rtest.Assert(t, fi.ModTime().Equal(time.Unix(0, 0)),
rtest.Assert(t, fi.ModTime() == time.Unix(0, 0),
"meta data of intermediate directory hasn't been restore")
}

Some files were not shown because too many files have changed in this diff Show More