Compare commits

..
Author SHA1 Message Date
Leo R. Lundgren 6d39410958 doc: Reword parts of the text, replace rclone with rest-server
The intent here is to make the text more consistent in its use of different
concepts involved in explaining the idea and setup that is explained, and
to make it easier to follow.

We're also replacing rclone with rest-server, not because we dislike rclone
but in order to keep the text to the basic tooling and the main restic eco-
system.

Finally we also remove the previous tip at the end about keeping the SSH tunnel
up, as it will be during the time the SSH session is running (in which the user
is expected to run the restic commands).
2026-02-18 22:19:56 +01:00
JL710 5c3116901e use rclone for rest server instead of docker 2026-01-17 16:29:26 +01:00
JL710 8943ca15ed apply suggestions from Michael Eischer 2025-09-25 13:35:40 +02:00
JL710 a9d51db68d add example for "Pulling a Backup with HTTP over a ssh tunnel" 2025-09-11 16:07:50 +02:00
557 changed files with 8100 additions and 15065 deletions
-25
View File
@@ -1,25 +0,0 @@
[codespell]
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
# Skip historical content (CHANGELOG, release notes archive), auto-generated
# man pages (which contain troff macros like \(bu that look like typos), and
# files we don't want to modify (go.sum, css with vendor prefixes).
skip = .git*,.gitignore,.gitattributes,go.sum,*.css,.codespellrc,CHANGELOG.md,changelog,*.1
check-hidden = true
# Protect URLs from corrections (URLs may contain words flagged as typos and
# must not be changed); also ignore camelCase/PascalCase identifiers like
# AtLeast which are valid Go identifiers, not typos.
ignore-regex = https?://\S+|\b[a-z]+[A-Z]\w*\b|\b[A-Z][a-z]+[A-Z]\w*\b
ignore-words-list =
# variable name short for "serialized", also matches [uU]ser glob pattern
ser,
# German word "ist" appearing in test data string
ist,
# intentional truncated test string in format truncation tests
fo,
# CLI flag name (--iinclude, case-insensitive include)
iinclude,
# intentional in fake test path /doesnt/exist
doesnt
-12
View File
@@ -1,12 +0,0 @@
# Actual layer caching is impossible due to .git, but
# that must be included for provenance reasons. These ignores
# are strictly for hygienic build.
*
!/*.go
!/go.*
!/cmd/*
!/docker/entrypoint.sh
!/internal/*
!/helpers/*
!/VERSION
!/.git/
+1 -1
View File
@@ -36,7 +36,7 @@ Please always follow these steps:
- Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits). - Format all commit messages in the same style as [the other commits in the repository](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
--> -->
- [ ] I have added tests for all code changes, see [writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests) - [ ] I have added tests for all code changes.
- [ ] I have added documentation for relevant changes (in the manual). - [ ] I have added documentation for relevant changes (in the manual).
- [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)). - [ ] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
- [ ] I'm done! This pull request is ready for review. - [ ] I'm done! This pull request is ready for review.
-4
View File
@@ -5,10 +5,6 @@ updates:
directory: "/" # Location of package manifests directory: "/" # Location of package manifests
schedule: schedule:
interval: "monthly" interval: "monthly"
groups:
golang-x-deps:
patterns:
- "golang.org/x/*"
# Dependencies listed in .github/workflows/*.yml # Dependencies listed in .github/workflows/*.yml
- package-ecosystem: "github-actions" - package-ecosystem: "github-actions"
-27
View File
@@ -1,27 +0,0 @@
# Codespell configuration is within .codespellrc
---
name: Codespell
on:
# run checks on push to master, but not when other branches are pushed to
push:
branches:
- master
# run checks for all pull requests
pull_request:
merge_group:
permissions:
contents: read
jobs:
codespell:
name: Check for spelling errors
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Codespell
uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2
+6 -6
View File
@@ -26,10 +26,10 @@ jobs:
steps: steps:
- name: Checkout repository - name: Checkout repository
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Log in to the Container registry - name: Log in to the Container registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1
with: with:
registry: ${{ env.REGISTRY }} registry: ${{ env.REGISTRY }}
username: ${{ github.actor }} username: ${{ github.actor }}
@@ -37,7 +37,7 @@ jobs:
- name: Extract metadata (tags, labels) for Docker - name: Extract metadata (tags, labels) for Docker
id: meta id: meta
uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
with: with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: | tags: |
@@ -47,10 +47,10 @@ jobs:
type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}}.{{minor}}
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
- name: Ensure consistent binaries - name: Ensure consistent binaries
run: | run: |
@@ -61,7 +61,7 @@ jobs:
- name: Build and push Docker image - name: Build and push Docker image
id: build-and-push id: build-and-push
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1
with: with:
push: true push: true
context: . context: .
+19 -21
View File
@@ -13,7 +13,7 @@ permissions:
contents: read contents: read
env: env:
latest_go: "1.26.x" latest_go: "1.24.x"
GO111MODULE: on GO111MODULE: on
jobs: jobs:
@@ -23,29 +23,29 @@ jobs:
# list of jobs to run: # list of jobs to run:
include: include:
- job_name: Windows - job_name: Windows
go: 1.26.x go: 1.24.x
os: windows-latest os: windows-latest
- job_name: macOS - job_name: macOS
go: 1.26.x go: 1.24.x
os: macOS-latest os: macOS-latest
test_fuse: false test_fuse: false
- job_name: Linux - job_name: Linux
go: 1.26.x go: 1.24.x
os: ubuntu-latest os: ubuntu-latest
test_cloud_backends: true test_cloud_backends: true
test_fuse: true test_fuse: true
check_changelog: true check_changelog: true
- job_name: Linux (race) - job_name: Linux (race)
go: 1.26.x go: 1.24.x
os: ubuntu-latest os: ubuntu-latest
test_fuse: true test_fuse: true
test_opts: "-race" test_opts: "-race"
- job_name: Linux - job_name: Linux
go: 1.25.x go: 1.23.x
os: ubuntu-latest os: ubuntu-latest
test_fuse: true test_fuse: true
@@ -57,10 +57,10 @@ jobs:
steps: steps:
- name: Check out code - name: Check out code
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up Go ${{ matrix.go }} - name: Set up Go ${{ matrix.go }}
uses: actions/setup-go@v6 uses: actions/setup-go@v5
with: with:
go-version: ${{ matrix.go }} go-version: ${{ matrix.go }}
@@ -137,14 +137,12 @@ jobs:
go run build.go go run build.go
- name: Minimal test - name: Minimal test
timeout-minutes: 3
run: | run: |
./restic init ./restic init
./restic backup . ./restic backup .
env: env:
RESTIC_REPOSITORY: ../testrepo RESTIC_REPOSITORY: ../testrepo
RESTIC_PASSWORD: password RESTIC_PASSWORD: password
RESTIC_DEBUG_STACKTRACE_SIGINT: "true"
- name: Run local Tests - name: Run local Tests
env: env:
@@ -222,10 +220,10 @@ jobs:
steps: steps:
- name: Check out code - name: Check out code
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up Go ${{ env.latest_go }} - name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v6 uses: actions/setup-go@v5
with: with:
go-version: ${{ env.latest_go }} go-version: ${{ env.latest_go }}
@@ -244,18 +242,18 @@ jobs:
checks: write checks: write
steps: steps:
- name: Check out code - name: Check out code
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Set up Go ${{ env.latest_go }} - name: Set up Go ${{ env.latest_go }}
uses: actions/setup-go@v6 uses: actions/setup-go@v5
with: with:
go-version: ${{ env.latest_go }} go-version: ${{ env.latest_go }}
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v6
with: with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v2.12 version: v1.64.8
args: --verbose --timeout 5m args: --verbose --timeout 5m
# only run golangci-lint for pull requests, otherwise ALL hints get # only run golangci-lint for pull requests, otherwise ALL hints get
@@ -289,11 +287,11 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Check out code - name: Check out code
uses: actions/checkout@v6 uses: actions/checkout@v5
- name: Docker meta - name: Docker meta
id: meta id: meta
uses: docker/metadata-action@v6 uses: docker/metadata-action@v5
with: with:
# list of Docker images to use as base name for tags # list of Docker images to use as base name for tags
images: | images: |
@@ -309,14 +307,14 @@ jobs:
type=sha type=sha
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@v4 uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4 uses: docker/setup-buildx-action@v3
- name: Build and push - name: Build and push
id: docker_build id: docker_build
uses: docker/build-push-action@v7 uses: docker/build-push-action@v6
with: with:
push: false push: false
context: . context: .
+56 -81
View File
@@ -1,95 +1,70 @@
version: "2" # This is the configuration for golangci-lint for the restic project.
#
# A sample config with all settings is here:
# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
linters: linters:
# only enable the linters listed below # only enable the linters listed below
default: none disable-all: true
enable: enable:
- asciicheck
# ensure that http response bodies are closed
- bodyclose
# restrict imports from other restic packages for internal/backend (cache exempt)
- depguard
- copyloopvar
# make sure all errors returned by functions are handled # make sure all errors returned by functions are handled
- errcheck - errcheck
# show how code can be simplified
- gosimple
# make sure code is formatted
- gofmt
# examine code and report suspicious constructs, such as Printf calls whose # examine code and report suspicious constructs, such as Printf calls whose
# arguments do not align with the format string # arguments do not align with the format string
- govet - govet
# consistent imports
- importas
# detect when assignments to existing variables are not used
- ineffassign
- nolintlint
# make sure names and comments are used according to the conventions # make sure names and comments are used according to the conventions
- revive - revive
# detect when assignments to existing variables are not used
- ineffassign
# run static analysis and find errors # run static analysis and find errors
- staticcheck - staticcheck
# find unused variables, functions, structs, types, etc. # find unused variables, functions, structs, types, etc.
- unused - unused
settings:
depguard: # parse and typecheck code
rules: - typecheck
# Prevent backend packages from importing the internal/restic package to keep the architectural layers intact.
backend-imports: # ensure that http response bodies are closed
files: - bodyclose
- "**/internal/backend/**"
- "!**/internal/backend/cache/**" - importas
- "!**/internal/backend/test/**"
- "!**/*_test.go" issues:
deny: # don't use the default exclude rules, this hides (among others) ignored
- pkg: "github.com/restic/restic/internal/restic" # errors from Close() calls
desc: "internal/restic should not be imported to keep the architectural layers intact" exclude-use-default: false
- pkg: "github.com/restic/restic/internal/repository"
desc: "internal/repository should not be imported to keep the architectural layers intact" # list of things to not warn about
importas: exclude:
alias: # revive: do not warn about missing comments for exported stuff
- pkg: github.com/restic/restic/internal/test - exported (function|method|var|type|const) .* should have comment or be unexported
alias: rtest # revive: ignore constants in all caps
staticcheck: - don't use ALL_CAPS in Go names; use CamelCase
checks: # revive: lots of packages don't have such a comment
# default - "package-comments: should have a package comment"
- "all" # staticcheck: there's no easy way to replace these packages
- "-ST1000" - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
- "-ST1003" - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
- "-ST1016" - "redefines-builtin-id:"
- "-ST1020"
- "-ST1021" exclude-rules:
- "-ST1022" # revive: ignore unused parameters in tests
# extra disables - path: (_test\.go|testing\.go|backend/.*/tests\.go)
- "-QF1008" # don't warn about specifying name of embedded field on access text: "unused-parameter:"
exclusions:
rules: linters-settings:
# revive: ignore unused parameters in tests importas:
- path: (_test\.go|testing\.go|backend/.*/tests\.go) alias:
text: "unused-parameter:" - pkg: github.com/restic/restic/internal/test
# revive: do not warn about missing comments for exported stuff alias: rtest
- path: (.+)\.go$
text: exported (function|method|var|type|const) .* should have comment or be unexported
# revive: ignore constants in all caps
- path: (.+)\.go$
text: don't use ALL_CAPS in Go names; use CamelCase
# revive: lots of packages don't have such a comment
- path: (.+)\.go$
text: "package-comments: should have a package comment"
# staticcheck: there's no easy way to replace these packages
- path: (.+)\.go$
text: 'SA1019: "golang.org/x/crypto/poly1305" is deprecated'
- path: (.+)\.go$
text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated'
- path: (.+)\.go$
text: "redefines-builtin-id:"
# revive: collection of helpers to implement a backend, more descriptive names would be too repetitive
- path: internal/backend/util/.*.go$
text: "var-naming: avoid meaningless package names"
paths:
- third_party$
- builtin$
- examples$
formatters:
enable:
# make sure code is formatted
- gofmt
exclusions:
paths:
- third_party$
- builtin$
- examples$
+2 -727
View File
@@ -1,8 +1,5 @@
# Table of Contents # Table of Contents
* [Changelog for 0.19.1](#changelog-for-restic-0191-2026-07-05)
* [Changelog for 0.19.0](#changelog-for-restic-0190-2026-06-09)
* [Changelog for 0.18.1](#changelog-for-restic-0181-2025-09-21)
* [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27) * [Changelog for 0.18.0](#changelog-for-restic-0180-2025-03-27)
* [Changelog for 0.17.3](#changelog-for-restic-0173-2024-11-08) * [Changelog for 0.17.3](#changelog-for-restic-0173-2024-11-08)
* [Changelog for 0.17.2](#changelog-for-restic-0172-2024-10-27) * [Changelog for 0.17.2](#changelog-for-restic-0172-2024-10-27)
@@ -42,727 +39,6 @@
* [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29)
# Changelog for restic 0.19.1 (2026-07-05)
The following sections list the changes in restic 0.19.1 relevant to
restic users. The changes are ordered by importance.
## Summary
* Fix #5234: Prevent mounting over the repository directory
* Fix #5667: Skip inaccessible `backup` source paths
* Fix #5722: Update `mount` latest symlink after snapshot reload
* Fix #21866: Hide `stats` progress bar in JSON mode
* Fix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
* Fix #21876: Show timezone location in `snapshots` output
* Fix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
* Fix #21895: Remove read-only files via the SFTP backend on Windows servers
* Fix #21899: Make `backup` respect excludes for duplicate directory entries
## Details
* Bugfix #5234: Prevent mounting over the repository directory
Using a local repository directory as the `mount` target — or a path that
contains it, or that it contains — caused the FUSE server to read its own
backend files through the new mount, deadlocking the kernel and requiring a long
reboot to recover.
Restic now resolves both paths and refuses any such overlap with a clear error
before mounting.
https://github.com/restic/restic/issues/5234
https://github.com/restic/restic/pull/5348
* Bugfix #5667: Skip inaccessible `backup` source paths
The `backup` command only skipped source paths that did not exist. A path that
could not be accessed for another reason, such as a malformed path on Windows,
was kept and produced an empty snapshot. Restic now skips any such path and
aborts if none remain.
https://github.com/restic/restic/issues/5667
https://github.com/restic/restic/pull/21852
* Bugfix #5722: Update `mount` latest symlink after snapshot reload
When `restic mount` was kept running while new snapshots were created, the new
snapshots appeared in the mountpoint, but the `latest` symlink could still point
to the previously latest snapshot. Restic now invalidates the cached snapshot
directory entries after a snapshot reload so that `latest` points to the newest
snapshot.
https://github.com/restic/restic/issues/5722
https://github.com/restic/restic/pull/21873
* Bugfix #21866: Hide `stats` progress bar in JSON mode
Since restic 0.19.0, the `stats` command shows a progress bar. This progress bar
was unintentionally displayed also when using the `--json` option, mixing
regular text output with JSON. This is now fixed.
https://github.com/restic/restic/issues/21866
https://github.com/restic/restic/pull/21871
* Bugfix #21869: Restore old behavior of `snapshots --latest <n>` without `--group-by`
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>` to
no longer group snapshots by host and paths by default.
The `snapshots --latest <n>` command now again uses the old behavior of grouping
by host and paths when `--group-by` is not specified. However, when specifying
`--group-by` the output is still grouped as requested, as in restic 0.19.0.
https://github.com/restic/restic/issues/21869
https://github.com/restic/restic/pull/21875
* Bugfix #21876: Show timezone location in `snapshots` output
With restic 0.19.0, the `snapshots` command printed the current timezone when
listing snapshots. However, that timezone label might change during the year,
for example with daylight saving time. Restic now prints a more consistent and
shorter version of the text.
https://github.com/restic/restic/pull/21876
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
* Bugfix #21879: Prevent crash in mountpoint validation if mountpoint is inaccessible
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
the repository. If restic was unable to stat the mountpoint, this would result
in a crash. This has now been fixed to correctly return an error instead.
https://github.com/restic/restic/pull/21879
* Bugfix #21895: Remove read-only files via the SFTP backend on Windows servers
Since restic 0.19.0, repository files on the SFTP backend are marked read-only
after save. On Windows SFTP servers, removing them failed with a permission
error. The SFTP backend now clears the read-only flag before removing the file.
https://github.com/restic/restic/issues/21895
https://github.com/restic/restic/pull/21897
* Bugfix #21899: Make `backup` respect excludes for duplicate directory entries
Since restic 0.19.0, backing up a directory with duplicate directory entries
always produced "Warning: at least one source file could not be read", even when
those files were excluded. This has now been fixed.
https://github.com/restic/restic/issues/21899
https://github.com/restic/restic/pull/21900
# Changelog for restic 0.19.0 (2026-06-09)
The following sections list the changes in restic 0.19.0 relevant to
restic users. The changes are ordered by importance.
## Summary
* Fix #2034: Support serving a `restic mount` of a Windows system via Samba
* Fix #4447: Use mode 0700 for repository directories created over SFTP
* Fix #4467: Exit with code 3 when some `backup` source paths do not exist
* Fix #4759: Error out when environment variables hold invalid values
* Fix #5233: Return exit code 3 when failing to remove snapshots
* Fix #5258: Exit with code 130 on SIGINT
* Fix #5280: Reject impossible `find` time bounds immediately
* Fix #5280: Make `find --pack` list blobs for tree packs
* Fix #5354: Allow `rclone` and `sftp` backends when running in background
* Fix #5427: Correctly restore ACL inheritance state on Windows
* Fix #5477: Password prompt was sometimes not shown for `backup -v`
* Fix #5487: Mark repository files read-only when using the SFTP backend
* Fix #5586: Correctly handle `snapshots --group-by` with `--latest`
* Fix #5595: Avoid spurious `chmod` errors on certain file backends
* Fix #5683: Prevent `backup --stdin-from-command` from hanging
* Fix #5757: Respect `--user` and `--host` in `key passwd`
* Fix #21820: Correct handling of duplicate index entries
* Fix #21820: Correctly handle pack files missing from the index
* Chg #5293: Prune small packfiles more aggressively
* Chg #5767: Prevent excluding paths explicitly passed to `backup`
* Chg #21791: Update dependencies and require Go 1.25 or newer
* Enh #3326: Limit `check` to snapshots selected by filters
* Enh #3572: Support restoring ownership by name on UNIX systems
* Enh #3738: Optional GitHub token for `self-update` API requests
* Enh #4278: Support include filters in the `rewrite` command
* Enh #4728: Support zstd compression levels `fastest` and `better`
* Enh #4868: Include repository ID in the filesystem name used by `mount`
* Enh #5175: Add status counters to `copy` in verbose text output
* Enh #5352: Support excluding cloud-backed files on macOS
* Enh #5383: Reduce progress bar refresh rates to decrease energy usage
* Enh #5424: Enable Windows filesystem privileges before file access
* Enh #5440: Make `--host` override environment variable `RESTIC_HOST`
* Enh #5448: Support configuring `nice` and `ionice` in the Docker image
* Enh #5453: Copy multiple snapshots in batches
* Enh #5523: Add Open Container Initiative labels to release Docker image
* Enh #5531: Reduce Azure storage costs by optimizing uploads
* Enh #5562: Rewrite only changed status lines each frame
* Enh #5588: Show timezone context in `snapshots` output
* Enh #5610: Reduce `check`, `copy`, `diff` and `stats` memory usage
* Enh #5689: Show more detailed progress for `stats`
* Enh #5713: Significantly speed up index loading
* Enh #5718: Stricter and earlier validation of the `mount` point
## Details
* Bugfix #2034: Support serving a `restic mount` of a Windows system via Samba
A repository mounted using `restic mount` on a POSIX system could not use Samba
to serve files from backups of Windows systems, while backups of non-Windows
systems could be served successfully. This has now been fixed.
https://github.com/restic/restic/issues/2034
https://github.com/restic/restic/issues/4382
https://github.com/restic/restic/pull/21784
* Bugfix #4447: Use mode 0700 for repository directories created over SFTP
When creating a repository over SFTP, restic created the repository directories
with the SFTP server's default permissions, often 0755, rather than the 0700
permissions it uses for local repositories.
Restic now creates these directories with 0700 permissions.
https://github.com/restic/restic/issues/4447
https://github.com/restic/restic/pull/21817
* Bugfix #4467: Exit with code 3 when some `backup` source paths do not exist
Restic used to exit with code 0 when a top-level backup source path was missing,
and exited with code 3 only when a child path under an existing source did not
exist. Scripts that relied on the exit code could therefore treat an incomplete
backup as success.
Restic now exits with code 3 when any backup source path does not exist.
https://github.com/restic/restic/issues/4467
https://github.com/restic/restic/pull/5347
* Bugfix #4759: Error out when environment variables hold invalid values
If the environment variables `RESTIC_COMPRESSION`, `RESTIC_PACK_SIZE`, or
`RESTIC_READ_CONCURRENCY` could not be parsed, restic used to ignore them.
Restic now fails with an error unless the same setting is overridden on the
command line.
https://github.com/restic/restic/issues/4759
https://github.com/restic/restic/pull/5592
https://github.com/restic/restic/pull/5700
* Bugfix #5233: Return exit code 3 when failing to remove snapshots
Previously, the `forget` command returned exit code 0 when it failed to remove
one or more snapshots. This was misleading to scripts.
The `forget` command now instead returns exit code 3 when failing to remove one
or more snapshots.
https://github.com/restic/restic/issues/5233
https://github.com/restic/restic/pull/5322
* Bugfix #5258: Exit with code 130 on SIGINT
Restic used to return exit code 1 on SIGINT. It now returns 130, the usual
convention for a process stopped by Ctrl-C.
https://github.com/restic/restic/issues/5258
https://github.com/restic/restic/pull/5363
* Bugfix #5280: Reject impossible `find` time bounds immediately
The `find` command now fails with an error when both `--oldest` and `--newest`
are set and `--oldest` is later than `--newest`.
https://github.com/restic/restic/issues/5280
https://github.com/restic/restic/pull/5310
* Bugfix #5280: Make `find --pack` list blobs for tree packs
The `find --pack <tree-pack>` command now also reports blobs for packs that only
contain tree blobs.
https://github.com/restic/restic/issues/5280
https://github.com/restic/restic/pull/5664
* Bugfix #5354: Allow `rclone` and `sftp` backends when running in background
Previously, starting restic in the background could result in unexpected
behavior when using the `rclone` or `sftp` backends. For example, `restic -r
rclone:./example --insecure-no-password init &` could cause the calling `bash`
shell to exit unexpectedly.
This has now been fixed.
https://github.com/restic/restic/issues/5354
https://github.com/restic/restic/pull/5358
https://github.com/restic/restic/pull/5493
https://github.com/restic/restic/pull/5494
* Bugfix #5427: Correctly restore ACL inheritance state on Windows
Since security descriptor backups were added in restic 0.17.0, Access Control
Entry inheritance was not restored correctly on Windows; restored permissions
were always marked as explicit (not inherited) even when they were inherited
from a parent folder.
The inheritance flags are now correctly applied when restoring the security
descriptor, preserving the original permission structure.
https://github.com/restic/restic/issues/5427
https://github.com/restic/restic/pull/5465
* Bugfix #5477: Password prompt was sometimes not shown for `backup -v`
The repository password prompt could be hidden when running the `backup -v`
command. This has now been fixed.
https://github.com/restic/restic/issues/5477
https://github.com/restic/restic/pull/5554
* Bugfix #5487: Mark repository files read-only when using the SFTP backend
Files created through the SFTP backend previously stayed writable. New files now
get read-only permissions where the server supports `chmod`.
https://github.com/restic/restic/issues/5487
https://github.com/restic/restic/pull/5497
* Bugfix #5586: Correctly handle `snapshots --group-by` with `--latest`
For the `snapshots` command, `--latest` did not interact correctly with a
non-default `--group-by` value. This combination now behaves as intended.
https://github.com/restic/restic/issues/5586
https://github.com/restic/restic/pull/5601
* Bugfix #5595: Avoid spurious `chmod` errors on certain file backends
On filesystems that do not support `chmod` (for example CIFS or FUSE-mounted
WebDAV), restic since version 0.18.0 failed to remove stale locks, throwing the
error `chmod ...: operation not supported`. This has now been fixed.
https://github.com/restic/restic/issues/5595
https://github.com/restic/restic/pull/5596
* Bugfix #5683: Prevent `backup --stdin-from-command` from hanging
When using `--stdin-from-command`, the `backup` command could hang until
manually cancelled if the backup stopped before all subprocess output was
consumed, for example after a failed upload to the repository. This has now been
fixed.
https://github.com/restic/restic/issues/5683
https://github.com/restic/restic/pull/21829
* Bugfix #5757: Respect `--user` and `--host` in `key passwd`
The `key passwd` command previously ignored the `--user` and `--host` options
and always stored the new key with the current user and host name. These options
are now honored.
https://github.com/restic/restic/issues/5757
https://github.com/restic/restic/pull/21781
* Bugfix #21820: Correct handling of duplicate index entries
Before restic 0.10.0, a bug could, in very rare cases, split information about a
pack file across multiple index files.
Since restic 0.17.0, any operation that rewrites the index (like `prune` or
`repair packs`) could lose part of that information, resulting in errors in
later `check` or `prune` runs. This can be fixed by running `repair packs`, and
only repositories using repository format version 1 might be affected.
Split pack index entries are no longer lost during index rewrites. The `check`
command now reports these cases as errors that can be fixed using the `repair
packs` command. On older restic versions, running `repair index` twice also
fixes the problem.
https://github.com/restic/restic/issues/21820
https://github.com/restic/restic/pull/21828
* Bugfix #21820: Correctly handle pack files missing from the index
The `repair packs` command was unable to salvage blobs from a pack file if the
pack file was not contained in the index or the index entry was incomplete.
The command now uses information from both the index and the pack file header.
https://github.com/restic/restic/issues/21820
https://github.com/restic/restic/pull/21827
* Change #5293: Prune small packfiles more aggressively
The `prune` command now repacks more small packfiles by default. The option
`--repack-small` is no longer needed and has been marked as deprecated. The
`--repack-smaller-than` option can still be used to further control repacking of
small pack files.
https://github.com/restic/restic/issues/5293
https://github.com/restic/restic/pull/21803
* Change #5767: Prevent excluding paths explicitly passed to `backup`
When e.g. `restic backup --exclude-if-present .git /home/user/data` was run and
`/home/user/.git` existed, restic excluded the `data` directory from the
snapshot. The same applied to `--exclude-caches` or `--one-file-system`.
Similarly, `restic backup --exclude-larger-than 1M large-file.bin` produced an
empty snapshot when the file was larger than one megabyte.
The `backup` command now tracks which files and directories were specified on
the command line and does not apply excludes to those paths. Content inside a
backed-up directory is still filtered by excludes as before.
https://github.com/restic/restic/issues/5767
https://github.com/restic/restic/pull/21797
* Change #21791: Update dependencies and require Go 1.25 or newer
Dependencies have been updated. Building restic now requires Go 1.25 or newer.
The Windows build with Go 1.26 was also fixed.
https://github.com/restic/restic/issues/21791
https://github.com/restic/restic/pull/5619
https://github.com/restic/restic/pull/21796
* Enhancement #3326: Limit `check` to snapshots selected by filters
The `check` command can now restrict pack verification to snapshots chosen with
the usual snapshot filters (`--tag`, `--host`, `--path`, or explicit snapshot
IDs on the command line).
https://github.com/restic/restic/issues/3326
https://github.com/restic/restic/pull/5469
https://github.com/restic/restic/pull/5644
* Enhancement #3572: Support restoring ownership by name on UNIX systems
The `restore` command used to restore file ownership on UNIX systems by UID and
GID. It now supports restoring ownership by user and group name with
`--ownership-by-name`, so that snapshots can be restored on systems where
numeric IDs do not match those on the backup host.
Note: POSIX ACLs are still restored by numeric value; this change does not add
ACL-by-name support.
https://github.com/restic/restic/issues/3572
https://github.com/restic/restic/pull/5449
* Enhancement #3738: Optional GitHub token for `self-update` API requests
The `self-update` command used only unauthenticated GitHub API requests when
checking for releases. Shared IP addresses could hit the GitHub rate limit,
resulting in a 403 Forbidden error and preventing updates.
Unauthenticated requests remain the default, but authenticated requests are now
possible. Set the environment variable `GITHUB_ACCESS_TOKEN` to a GitHub
[personal access token](https://github.com/settings/tokens) to avoid rate-limit
failures.
https://github.com/restic/restic/issues/3738
https://github.com/restic/restic/pull/5568
* Enhancement #4278: Support include filters in the `rewrite` command
The `rewrite` command now accepts the same include filter options as the
`restore` command (`--include`, `--include-file`, `--iinclude`,
`--iinclude-file`, and short `-i`). Include and exclude filter options are
mutually exclusive.
https://github.com/restic/restic/issues/4278
https://github.com/restic/restic/pull/5191
* Enhancement #4728: Support zstd compression levels `fastest` and `better`
Restic now supports the zstd compression modes `fastest` and `better`. Set the
environment variable `RESTIC_COMPRESSION` to `fastest` or `better`, or pass the
same values with the `--compression` option.
https://github.com/restic/restic/issues/4728
https://github.com/restic/restic/pull/5321
* Enhancement #4868: Include repository ID in the filesystem name used by `mount`
The filesystem exposed by the `mount` command now includes the repository ID in
its name. The ID is printed when opening a repository or can be read with
`restic cat config`.
```
$ df ./test-mount/
Filesystem 1K-blocks Used Available Use% Mounted on
restic:d3b07384d1 0 0 0 - /mnt/my-restic-repo
```
https://github.com/restic/restic/issues/4868
https://github.com/restic/restic/pull/5243
* Enhancement #5175: Add status counters to `copy` in verbose text output
The `copy` command now prints additional counters in text mode when `--verbose`
is set: blobs to copy, their on-disk size, and the number of pack files read
from the source repository.
https://github.com/restic/restic/issues/5175
https://github.com/restic/restic/pull/5319
* Enhancement #5352: Support excluding cloud-backed files on macOS
Previously, restic treated cloud-backed files (such as files stored on iCloud)
like normal local files, forcing a full download of placeholders and other
"meant to be cloud only" content during backups.
The `backup` command now supports `--exclude-cloud-files` (previously only
available on Windows) to skip those files on supported macOS versions. From
Sonoma (macOS 14.0) onward the option can prevent unwanted downloads. Older
macOS versions will still download the files during a backup run.
https://github.com/restic/restic/issues/5352
https://github.com/restic/restic/pull/5370
* Enhancement #5383: Reduce progress bar refresh rates to decrease energy usage
Progress bars were previously updated at 60 frames per second, which could cause
high CPU or GPU usage in some terminal emulators.
The refresh rate is now 10 FPS to conserve energy. For some terminal emulators,
the lower rate is also necessary to allow selecting text in the terminal.
https://github.com/restic/restic/issues/5383
https://github.com/restic/restic/pull/5551
https://github.com/restic/restic/pull/5626
* Enhancement #5424: Enable Windows filesystem privileges before file access
Restic used to enable some Windows filesystem privileges only while reading or
writing security descriptors. Extended attributes could therefore be read before
enabling the backup privilege, possibly resulting in missed data or errors.
Restic now enables the relevant filesystem privileges before any file access.
https://github.com/restic/restic/pull/5424
* Enhancement #5440: Make `--host` override environment variable `RESTIC_HOST`
Previously, when the environment variable `RESTIC_HOST` was set, snapshot
listings and other operations were always filtered to that host.
Passing `--host` as an empty string (`--host=""` or `--host=`) now overrides
`RESTIC_HOST` and shows snapshots from all hosts.
The same override applies to other commands that support snapshot filters,
including `snapshots`, `forget`, `find`, `stats`, `copy`, `tag`, `repair
snapshots`, `rewrite`, `mount`, `restore`, `dump`, and `ls`.
https://github.com/restic/restic/issues/5440
https://github.com/restic/restic/pull/5541
* Enhancement #5448: Support configuring `nice` and `ionice` in the Docker image
The container entrypoint now reads optional scheduling hints from the
environment:
- The environment variable `NICE` sets the process nice value (see `man nice`).
- The environment variable `IONICE_CLASS` selects the I/O scheduling class (see
`man ionice`). Real-time classes need the `SYS_NICE` capability added to the
container.
- The environment variable `IONICE_PRIORITY` sets the priority within
`IONICE_CLASS` and has no effect unless `IONICE_CLASS` is set; it defaults to
`4` (neutral priority).
For further details, please see:
https://restic.readthedocs.io/en/stable/020_installation.html#docker-container
https://github.com/restic/restic/pull/5448
* Enhancement #5453: Copy multiple snapshots in batches
The `copy` command used to copy snapshots one at a time, even when doing so
produced pack files smaller than the target pack size. This led to many small
files when copying small incremental snapshots.
The `copy` command now copies multiple snapshots together so that small pack
files are avoided where possible.
https://github.com/restic/restic/issues/5453
https://github.com/restic/restic/pull/5472
* Enhancement #5523: Add Open Container Initiative labels to release Docker image
The release Docker image now includes OCI-style image annotation labels, which
helps external tooling identify the image.
https://github.com/restic/restic/pull/5523
* Enhancement #5531: Reduce Azure storage costs by optimizing uploads
Restic previously used Azure PutBlock and PutBlockList for every upload, which
cost two storage transactions per file and roughly doubled transaction charges
for repositories with many pack files.
Files up to 256 MiB now use PutBlob, requiring only a single transaction per
file and cutting typical transaction costs by about half. Larger blobs still use
block uploads as required by Azure.
https://github.com/restic/restic/issues/5531
https://github.com/restic/restic/pull/5544
* Enhancement #5562: Rewrite only changed status lines each frame
The status bar rewrote every line on each frame whenever any content changed,
which made selecting text impossible in some terminal emulators even when most
lines were unchanged.
Now only lines that actually change are rewritten on each update.
https://github.com/restic/restic/issues/5562
https://github.com/restic/restic/pull/5648
* Enhancement #5588: Show timezone context in `snapshots` output
The `snapshots` command now prints which timezone is used for displayed
timestamps. Snapshots may have been created in different timezones but are shown
in the local timezone, so a footer line (for example, timestamps shown in CET)
clarifies the display context when comparing snapshots from several sources.
https://github.com/restic/restic/pull/5588
* Enhancement #5610: Reduce `check`, `copy`, `diff` and `stats` memory usage
The `check`, `copy`, `diff` and `stats` commands now use less memory when
handling large snapshots.
https://github.com/restic/restic/pull/5610
* Enhancement #5689: Show more detailed progress for `stats`
The `stats` command used to show progress only while loading the index. During
the scan it printed only `scanning...` with no further updates. It now reports
how many snapshots, files, and blobs have been processed so far.
https://github.com/restic/restic/issues/5689
https://github.com/restic/restic/pull/5705
* Enhancement #5713: Significantly speed up index loading
Loading the index for a large repository is now significantly faster. Also, the
`mount` command now loads the index once at startup and then only loads new
index files as they appear. It also loads snapshots before printing that the
repository is being served.
https://github.com/restic/restic/pull/5713
https://github.com/restic/restic/pull/5720
* Enhancement #5718: Stricter and earlier validation of the `mount` point
The `mount` command previously accepted invalid mount points, resulting in an
error after loading the repository. The specified mount point must now refer to
a directory that the current user can access and write to, and this check is
performed before opening the repository.
https://github.com/restic/restic/pull/5718
# Changelog for restic 0.18.1 (2025-09-21)
The following sections list the changes in restic 0.18.1 relevant to
restic users. The changes are ordered by importance.
## Summary
* Fix #5324: Correctly handle `backup --stdin-filename` with directory paths
* Fix #5325: Accept `RESTIC_HOST` environment variable in `forget` command
* Fix #5342: Ignore "chmod not supported" errors when writing files
* Fix #5344: Ignore `EOPNOTSUPP` errors for extended attributes
* Fix #5421: Fix rare crash if directory is removed during backup
* Fix #5429: Stop retrying uploads when rest-server runs out of space
* Fix #5467: Improve handling of download retries in `check` command
## Details
* Bugfix #5324: Correctly handle `backup --stdin-filename` with directory paths
In restic 0.18.0, the `backup` command failed if a filename that includes at
least a directory was passed to `--stdin-filename`. For example,
`--stdin-filename /foo/bar` resulted in the following error:
```
Fatal: unable to save snapshot: open /foo: no such file or directory
```
This has now been fixed.
https://github.com/restic/restic/issues/5324
https://github.com/restic/restic/pull/5356
* Bugfix #5325: Accept `RESTIC_HOST` environment variable in `forget` command
The `forget` command did not use the host name from the `RESTIC_HOST`
environment variable when filtering snapshots. This has now been fixed.
https://github.com/restic/restic/issues/5325
https://github.com/restic/restic/pull/5327
* Bugfix #5342: Ignore "chmod not supported" errors when writing files
Restic 0.18.0 introduced a bug that caused `chmod xxx: operation not supported`
errors to appear when writing to a local file repository that did not support
chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors.
https://github.com/restic/restic/issues/5342
* Bugfix #5344: Ignore `EOPNOTSUPP` errors for extended attributes
Restic 0.18.0 added extended attribute support for NetBSD 10+, but not all
NetBSD filesystems support extended attributes. Other BSD systems can likewise
return `EOPNOTSUPP`, so restic now ignores these errors.
https://github.com/restic/restic/issues/5344
* Bugfix #5421: Fix rare crash if directory is removed during backup
In restic 0.18.0, the `backup` command could crash if a directory was removed
between reading its metadata and listing its directory content. This has now
been fixed.
https://github.com/restic/restic/pull/5421
* Bugfix #5429: Stop retrying uploads when rest-server runs out of space
When rest-server returns a `507 Insufficient Storage` error, it indicates that
no more storage capacity is available. Restic now correctly stops retrying
uploads in this case.
https://github.com/restic/restic/issues/5429
https://github.com/restic/restic/pull/5452
* Bugfix #5467: Improve handling of download retries in `check` command
In very rare cases, the `check` command could unnecessarily report repository
damage if the backend returned incomplete, corrupted data on the first download
try which is afterwards resolved by a download retry.
This could result in an error output like the following:
```
Load(<data/34567890ab>, 33918928, 0) returned error, retrying after 871.35598ms: readFull: unexpected EOF
Load(<data/34567890ab>, 33918928, 0) operation successful after 1 retries
check successful on second attempt, original error pack 34567890ab[...] contains 6 errors: [blob 12345678[...]: decrypting blob <data/12345678> from 34567890 failed: ciphertext verification failed ...]
[...]
Fatal: repository contains errors
```
This fix only applies to a very specific case where the log shows `operation
successful after 1 retries` followed by a `check successful on second attempt,
original error` that only reports `ciphertext verification failed` errors in the
pack file. If any other errors are reported in the pack file, then the
repository still has to be considered as damaged.
Now, only the check result of the last download retry is reported as intended.
https://github.com/restic/restic/issues/5467
https://github.com/restic/restic/pull/5495
# Changelog for restic 0.18.0 (2025-03-27) # Changelog for restic 0.18.0 (2025-03-27)
The following sections list the changes in restic 0.18.0 relevant to The following sections list the changes in restic 0.18.0 relevant to
restic users. The changes are ordered by importance. restic users. The changes are ordered by importance.
@@ -1330,9 +606,8 @@ restic users. The changes are ordered by importance.
correctly. correctly.
Specifying volume names is now handled correctly. To restore snapshots created Specifying volume names is now handled correctly. To restore snapshots created
before this bugfix, use the <snapshot>:<subfolder> syntax. For example, to before this bugfix, use the <snapshot>:<subpath> syntax. For example, to restore
restore a snapshot with ID `12345678` that backed up `C:`, use the following a snapshot with ID `12345678` that backed up `C:`, use the following command:
command:
``` ```
restic restore 12345678:/C/C:./ --target output/folder restic restore 12345678:/C/C:./ --target output/folder
-3
View File
@@ -202,9 +202,6 @@ we'll be glad to assist. Having a PR with failing integration tests is nothing
to be ashamed of. In contrast, that happens regularly for all of us. That's to be ashamed of. In contrast, that happens regularly for all of us. That's
what the tests are there for. what the tests are there for.
More details of how to structure tests can be found here at
[writing tests](https://restic.readthedocs.io/en/stable/090_participating.html#writing-tests).
Git Commits Git Commits
----------- -----------
+1 -1
View File
@@ -1 +1 @@
0.19.1-dev 0.18.0-dev
+2 -1
View File
@@ -36,6 +36,7 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//go:build ignore_build_go //go:build ignore_build_go
// +build ignore_build_go
package main package main
@@ -59,7 +60,7 @@ var config = Config{
// see https://github.com/googleapis/google-cloud-go/issues/11448 // see https://github.com/googleapis/google-cloud-go/issues/11448
DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used DefaultBuildTags: []string{"selfupdate", "disable_grpc_modules"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run Tests: []string{"./..."}, // tests to run
MinVersion: GoVersion{Major: 1, Minor: 25, Patch: 0}, // minimum Go version supported MinVersion: GoVersion{Major: 1, Minor: 23, Patch: 0}, // minimum Go version supported
} }
// Config configures the build. // Config configures the build.
+1 -1
View File
@@ -6,7 +6,7 @@ resulting snapshot would result in an error. Note that using `C:\`
as backup target worked correctly. as backup target worked correctly.
Specifying volume names is now handled correctly. To restore snapshots Specifying volume names is now handled correctly. To restore snapshots
created before this bugfix, use the <snapshot>:<subfolder> syntax. For created before this bugfix, use the <snapshot>:<subpath> syntax. For
example, to restore a snapshot with ID `12345678` that backed up `C:`, example, to restore a snapshot with ID `12345678` that backed up `C:`,
use the following command: use the following command:
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Ignore `EOPNOTSUPP` errors for extended attributes
Restic 0.18.0 added extended attribute support for NetBSD 10+, but not all
NetBSD filesystems support extended attributes. Other BSD systems can
likewise return `EOPNOTSUPP`, so restic now ignores these errors.
https://github.com/restic/restic/issues/5344
-8
View File
@@ -1,8 +0,0 @@
Bugfix: Stop retrying uploads when rest-server runs out of space
When rest-server returns a `507 Insufficient Storage` error, it indicates
that no more storage capacity is available. Restic now correctly stops
retrying uploads in this case.
https://github.com/restic/restic/issues/5429
https://github.com/restic/restic/pull/5452
-27
View File
@@ -1,27 +0,0 @@
Bugfix: Improve handling of download retries in `check` command
In very rare cases, the `check` command could unnecessarily report repository
damage if the backend returned incomplete, corrupted data on the first download
try which is afterwards resolved by a download retry.
This could result in an error output like the following:
```
Load(<data/34567890ab>, 33918928, 0) returned error, retrying after 871.35598ms: readFull: unexpected EOF
Load(<data/34567890ab>, 33918928, 0) operation successful after 1 retries
check successful on second attempt, original error pack 34567890ab[...] contains 6 errors: [blob 12345678[...]: decrypting blob <data/12345678> from 34567890 failed: ciphertext verification failed ...]
[...]
Fatal: repository contains errors
```
This fix only applies to a very specific case where the log shows
`operation successful after 1 retries` followed by a
`check successful on second attempt, original error` that only reports
`ciphertext verification failed` errors in the pack file. If any other errors
are reported in the pack file, then the repository still has to be considered
as damaged.
Now, only the check result of the last download retry is reported as intended.
https://github.com/restic/restic/issues/5467
https://github.com/restic/restic/pull/5495
-17
View File
@@ -1,17 +0,0 @@
Bugfix: Correct handling of duplicate index entries
Before restic 0.10.0, a bug could, in very rare cases, split information
about a pack file across multiple index files.
Since restic 0.17.0, any operation that rewrites the index (like `prune`
or `repair packs`) could lose part of that information, resulting in errors
in later `check` or `prune` runs. This can be fixed by running `repair packs`,
and only repositories using repository format version 1 might be affected.
Split pack index entries are no longer lost during index rewrites. The
`check` command now reports these cases as errors that can be fixed using
the `repair packs` command. On older restic versions, running `repair index`
twice also fixes the problem.
https://github.com/restic/restic/issues/21820
https://github.com/restic/restic/pull/21828
-9
View File
@@ -1,9 +0,0 @@
Enhancement: Limit `check` to snapshots selected by filters
The `check` command can now restrict pack verification to snapshots chosen with
the usual snapshot filters (`--tag`, `--host`, `--path`, or explicit snapshot
IDs on the command line).
https://github.com/restic/restic/issues/3326
https://github.com/restic/restic/pull/5469
https://github.com/restic/restic/pull/5644
-12
View File
@@ -1,12 +0,0 @@
Enhancement: Support restoring ownership by name on UNIX systems
The `restore` command used to restore file ownership on UNIX systems by UID
and GID. It now supports restoring ownership by user and group name with
`--ownership-by-name`, so that snapshots can be restored on systems where
numeric IDs do not match those on the backup host.
Note: POSIX ACLs are still restored by numeric value; this change does not
add ACL-by-name support.
https://github.com/restic/restic/issues/3572
https://github.com/restic/restic/pull/5449
-13
View File
@@ -1,13 +0,0 @@
Enhancement: Optional GitHub token for `self-update` API requests
The `self-update` command used only unauthenticated GitHub API requests when
checking for releases. Shared IP addresses could hit the GitHub rate limit,
resulting in a 403 Forbidden error and preventing updates.
Unauthenticated requests remain the default, but authenticated requests are now
possible. Set the environment variable `GITHUB_ACCESS_TOKEN` to a GitHub
[personal access token](https://github.com/settings/tokens) to avoid rate-limit
failures.
https://github.com/restic/restic/issues/3738
https://github.com/restic/restic/pull/5568
-8
View File
@@ -1,8 +0,0 @@
Enhancement: Support include filters in the `rewrite` command
The `rewrite` command now accepts the same include filter options as the `restore`
command (`--include`, `--include-file`, `--iinclude`, `--iinclude-file`, and
short `-i`). Include and exclude filter options are mutually exclusive.
https://github.com/restic/restic/issues/4278
https://github.com/restic/restic/pull/5191
-10
View File
@@ -1,10 +0,0 @@
Bugfix: Use mode 0700 for repository directories created over SFTP
When creating a repository over SFTP, restic created the repository directories
with the SFTP server's default permissions, often 0755, rather than the 0700
permissions it uses for local repositories.
Restic now creates these directories with 0700 permissions.
https://github.com/restic/restic/issues/4447
https://github.com/restic/restic/pull/21817
-11
View File
@@ -1,11 +0,0 @@
Bugfix: Exit with code 3 when some `backup` source paths do not exist
Restic used to exit with code 0 when a top-level backup source path was
missing, and exited with code 3 only when a child path under an existing
source did not exist. Scripts that relied on the exit code could therefore
treat an incomplete backup as success.
Restic now exits with code 3 when any backup source path does not exist.
https://github.com/restic/restic/issues/4467
https://github.com/restic/restic/pull/5347
-10
View File
@@ -1,10 +0,0 @@
Bugfix: Error out when environment variables hold invalid values
If the environment variables `RESTIC_COMPRESSION`, `RESTIC_PACK_SIZE`, or
`RESTIC_READ_CONCURRENCY` could not be parsed, restic used to ignore them.
Restic now fails with an error unless the same setting is overridden on the
command line.
https://github.com/restic/restic/issues/4759
https://github.com/restic/restic/pull/5592
https://github.com/restic/restic/pull/5700
-14
View File
@@ -1,14 +0,0 @@
Enhancement: Include repository ID in the filesystem name used by `mount`
The filesystem exposed by the `mount` command now includes the repository ID
in its name. The ID is printed when opening a repository or can be read with
`restic cat config`.
```
$ df ./test-mount/
Filesystem 1K-blocks Used Available Use% Mounted on
restic:d3b07384d1 0 0 0 - /mnt/my-restic-repo
```
https://github.com/restic/restic/issues/4868
https://github.com/restic/restic/pull/5243
-10
View File
@@ -1,10 +0,0 @@
Bugfix: Return exit code 3 when failing to remove snapshots
Previously, the `forget` command returned exit code 0 when it failed
to remove one or more snapshots. This was misleading to scripts.
The `forget` command now instead returns exit code 3 when failing to
remove one or more snapshots.
https://github.com/restic/restic/issues/5233
https://github.com/restic/restic/pull/5322
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Exit with code 130 on SIGINT
Restic used to return exit code 1 on SIGINT. It now returns 130, the usual
convention for a process stopped by Ctrl-C.
https://github.com/restic/restic/issues/5258
https://github.com/restic/restic/pull/5363
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Reject impossible `find` time bounds immediately
The `find` command now fails with an error when both `--oldest`
and `--newest` are set and `--oldest` is later than `--newest`.
https://github.com/restic/restic/issues/5280
https://github.com/restic/restic/pull/5310
-9
View File
@@ -1,9 +0,0 @@
Change: Prune small packfiles more aggressively
The `prune` command now repacks more small packfiles by default. The option
`--repack-small` is no longer needed and has been marked as deprecated. The
`--repack-smaller-than` option can still be used to further control repacking
of small pack files.
https://github.com/restic/restic/issues/5293
https://github.com/restic/restic/pull/21803
-13
View File
@@ -1,13 +0,0 @@
Enhancement: Support excluding cloud-backed files on macOS
Previously, restic treated cloud-backed files (such as files stored on iCloud)
like normal local files, forcing a full download of placeholders and other
"meant to be cloud only" content during backups.
The `backup` command now supports `--exclude-cloud-files` (previously only
available on Windows) to skip those files on supported macOS versions. From
Sonoma (macOS 14.0) onward the option can prevent unwanted downloads. Older
macOS versions will still download the files during a backup run.
https://github.com/restic/restic/issues/5352
https://github.com/restic/restic/pull/5370
-13
View File
@@ -1,13 +0,0 @@
Bugfix: Allow `rclone` and `sftp` backends when running in background
Previously, starting restic in the background could result in unexpected
behavior when using the `rclone` or `sftp` backends. For example,
`restic -r rclone:./example --insecure-no-password init &` could cause the
calling `bash` shell to exit unexpectedly.
This has now been fixed.
https://github.com/restic/restic/issues/5354
https://github.com/restic/restic/pull/5358
https://github.com/restic/restic/pull/5493
https://github.com/restic/restic/pull/5494
-12
View File
@@ -1,12 +0,0 @@
Enhancement: Reduce progress bar refresh rates to decrease energy usage
Progress bars were previously updated at 60 frames per second, which
could cause high CPU or GPU usage in some terminal emulators.
The refresh rate is now 10 FPS to conserve energy. For some terminal
emulators, the lower rate is also necessary to allow selecting text
in the terminal.
https://github.com/restic/restic/issues/5383
https://github.com/restic/restic/pull/5551
https://github.com/restic/restic/pull/5626
-14
View File
@@ -1,14 +0,0 @@
Enhancement: Make `--host` override environment variable `RESTIC_HOST`
Previously, when the environment variable `RESTIC_HOST` was set, snapshot
listings and other operations were always filtered to that host.
Passing `--host` as an empty string (`--host=""` or `--host=`) now overrides
`RESTIC_HOST` and shows snapshots from all hosts.
The same override applies to other commands that support snapshot filters,
including `snapshots`, `forget`, `find`, `stats`, `copy`, `tag`,
`repair snapshots`, `rewrite`, `mount`, `restore`, `dump`, and `ls`.
https://github.com/restic/restic/issues/5440
https://github.com/restic/restic/pull/5541
-11
View File
@@ -1,11 +0,0 @@
Enhancement: Copy multiple snapshots in batches
The `copy` command used to copy snapshots one at a time, even when doing so
produced pack files smaller than the target pack size. This led to many small
files when copying small incremental snapshots.
The `copy` command now copies multiple snapshots together so that small pack
files are avoided where possible.
https://github.com/restic/restic/issues/5453
https://github.com/restic/restic/pull/5472
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Password prompt was sometimes not shown for `backup -v`
The repository password prompt could be hidden when running the
`backup -v` command. This has now been fixed.
https://github.com/restic/restic/issues/5477
https://github.com/restic/restic/pull/5554
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Mark repository files read-only when using the SFTP backend
Files created through the SFTP backend previously stayed writable. New files now
get read-only permissions where the server supports `chmod`.
https://github.com/restic/restic/issues/5487
https://github.com/restic/restic/pull/5497
-12
View File
@@ -1,12 +0,0 @@
Enhancement: Reduce Azure storage costs by optimizing uploads
Restic previously used Azure PutBlock and PutBlockList for every upload, which
cost two storage transactions per file and roughly doubled transaction charges
for repositories with many pack files.
Files up to 256 MiB now use PutBlob, requiring only a single transaction per
file and cutting typical transaction costs by about half. Larger blobs still
use block uploads as required by Azure.
https://github.com/restic/restic/issues/5531
https://github.com/restic/restic/pull/5544
-10
View File
@@ -1,10 +0,0 @@
Enhancement: Rewrite only changed status lines each frame
The status bar rewrote every line on each frame whenever any content changed,
which made selecting text impossible in some terminal emulators even when most
lines were unchanged.
Now only lines that actually change are rewritten on each update.
https://github.com/restic/restic/issues/5562
https://github.com/restic/restic/pull/5648
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Correctly handle `snapshots --group-by` with `--latest`
For the `snapshots` command, `--latest` did not interact correctly with a
non-default `--group-by` value. This combination now behaves as intended.
https://github.com/restic/restic/issues/5586
https://github.com/restic/restic/pull/5601
-8
View File
@@ -1,8 +0,0 @@
Bugfix: Avoid spurious `chmod` errors on certain file backends
On filesystems that do not support `chmod` (for example CIFS or FUSE-mounted
WebDAV), restic since version 0.18.0 failed to remove stale locks, throwing
the error `chmod ...: operation not supported`. This has now been fixed.
https://github.com/restic/restic/issues/5595
https://github.com/restic/restic/pull/5596
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Prevent `backup --stdin-from-command` from hanging
When using `--stdin-from-command`, the `backup` command could hang until
manually cancelled if the backup stopped before all subprocess output was
consumed, for example after a failed upload to the repository. This has
now been fixed.
https://github.com/restic/restic/issues/5683
https://github.com/restic/restic/pull/21829
-8
View File
@@ -1,8 +0,0 @@
Enhancement: Show more detailed progress for `stats`
The `stats` command used to show progress only while loading the index. During
the scan it printed only `scanning...` with no further updates. It now reports
how many snapshots, files, and blobs have been processed so far.
https://github.com/restic/restic/issues/5689
https://github.com/restic/restic/pull/5705
-8
View File
@@ -1,8 +0,0 @@
Bugfix: Respect `--user` and `--host` in `key passwd`
The `key passwd` command previously ignored the `--user` and
`--host` options and always stored the new key with the current
user and host name. These options are now honored.
https://github.com/restic/restic/issues/5757
https://github.com/restic/restic/pull/21781
-15
View File
@@ -1,15 +0,0 @@
Change: Prevent excluding paths explicitly passed to `backup`
When e.g. `restic backup --exclude-if-present .git /home/user/data` was run
and `/home/user/.git` existed, restic excluded the `data` directory from the
snapshot. The same applied to `--exclude-caches` or `--one-file-system`.
Similarly, `restic backup --exclude-larger-than 1M large-file.bin` produced
an empty snapshot when the file was larger than one megabyte.
The `backup` command now tracks which files and directories were specified on
the command line and does not apply excludes to those paths. Content inside a
backed-up directory is still filtered by excludes as before.
https://github.com/restic/restic/issues/5767
https://github.com/restic/restic/pull/21797
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Support serving a `restic mount` of a Windows system via Samba
A repository mounted using `restic mount` on a POSIX system could not use
Samba to serve files from backups of Windows systems, while backups of
non-Windows systems could be served successfully. This has now been fixed.
https://github.com/restic/restic/issues/2034
https://github.com/restic/restic/issues/4382
https://github.com/restic/restic/pull/21784
-8
View File
@@ -1,8 +0,0 @@
Change: Update dependencies and require Go 1.25 or newer
Dependencies have been updated. Building restic now requires
Go 1.25 or newer. The Windows build with Go 1.26 was also fixed.
https://github.com/restic/restic/issues/21791
https://github.com/restic/restic/pull/5619
https://github.com/restic/restic/pull/21796
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Correctly handle pack files missing from the index
The `repair packs` command was unable to salvage blobs from a pack file if
the pack file was not contained in the index or the index entry was incomplete.
The command now uses information from both the index and the pack file header.
https://github.com/restic/restic/issues/21820
https://github.com/restic/restic/pull/21827
-8
View File
@@ -1,8 +0,0 @@
Enhancement: Add status counters to `copy` in verbose text output
The `copy` command now prints additional counters in text mode when
`--verbose` is set: blobs to copy, their on-disk size, and the number
of pack files read from the source repository.
https://github.com/restic/restic/issues/5175
https://github.com/restic/restic/pull/5319
-10
View File
@@ -1,10 +0,0 @@
Enhancement: Enable Windows filesystem privileges before file access
Restic used to enable some Windows filesystem privileges only while reading
or writing security descriptors. Extended attributes could therefore be read
before enabling the backup privilege, possibly resulting in missed data or
errors.
Restic now enables the relevant filesystem privileges before any file access.
https://github.com/restic/restic/pull/5424
-19
View File
@@ -1,19 +0,0 @@
Enhancement: Support configuring `nice` and `ionice` in the Docker image
The container entrypoint now reads optional scheduling hints from the
environment:
- The environment variable `NICE` sets the process nice value (see `man nice`).
- The environment variable `IONICE_CLASS` selects the I/O scheduling class (see
`man ionice`). Real-time classes need the `SYS_NICE` capability added to the
container.
- The environment variable `IONICE_PRIORITY` sets the priority within
`IONICE_CLASS` and has no effect unless `IONICE_CLASS` is set; it defaults to
`4` (neutral priority).
For further details, please see:
https://restic.readthedocs.io/en/stable/020_installation.html#docker-container
https://github.com/restic/restic/pull/5448
-12
View File
@@ -1,12 +0,0 @@
Bugfix: Correctly restore ACL inheritance state on Windows
Since security descriptor backups were added in restic 0.17.0, Access Control
Entry inheritance was not restored correctly on Windows; restored permissions
were always marked as explicit (not inherited) even when they were inherited
from a parent folder.
The inheritance flags are now correctly applied when restoring the security
descriptor, preserving the original permission structure.
https://github.com/restic/restic/issues/5427
https://github.com/restic/restic/pull/5465
-6
View File
@@ -1,6 +0,0 @@
Enhancement: Add Open Container Initiative labels to release Docker image
The release Docker image now includes OCI-style image annotation labels,
which helps external tooling identify the image.
https://github.com/restic/restic/pull/5523
-9
View File
@@ -1,9 +0,0 @@
Enhancement: Show timezone context in `snapshots` output
The `snapshots` command now prints which timezone is used for displayed
timestamps. Snapshots may have been created in different timezones but are
shown in the local timezone, so a footer line (for example, timestamps shown in
CET) clarifies the display context when comparing snapshots from several
sources.
https://github.com/restic/restic/pull/5588
-6
View File
@@ -1,6 +0,0 @@
Enhancement: Reduce `check`, `copy`, `diff` and `stats` memory usage
The `check`, `copy`, `diff` and `stats` commands now use less memory
when handling large snapshots.
https://github.com/restic/restic/pull/5610
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Make `find --pack` list blobs for tree packs
The `find --pack <tree-pack>` command now also reports
blobs for packs that only contain tree blobs.
https://github.com/restic/restic/issues/5280
https://github.com/restic/restic/pull/5664
-8
View File
@@ -1,8 +0,0 @@
Enhancement: Stricter and earlier validation of the `mount` point
The `mount` command previously accepted invalid mount points, resulting in an
error after loading the repository. The specified mount point must now refer to
a directory that the current user can access and write to, and this check is
performed before opening the repository.
https://github.com/restic/restic/pull/5718
-9
View File
@@ -1,9 +0,0 @@
Enhancement: Significantly speed up index loading
Loading the index for a large repository is now significantly faster. Also,
the `mount` command now loads the index once at startup and then only loads
new index files as they appear. It also loads snapshots before printing that
the repository is being served.
https://github.com/restic/restic/pull/5713
https://github.com/restic/restic/pull/5720
-8
View File
@@ -1,8 +0,0 @@
Bugfix: Hide `stats` progress bar in JSON mode
Since restic 0.19.0, the `stats` command shows a progress bar. This
progress bar was unintentionally displayed also when using the `--json`
option, mixing regular text output with JSON. This is now fixed.
https://github.com/restic/restic/issues/21866
https://github.com/restic/restic/pull/21871
-12
View File
@@ -1,12 +0,0 @@
Bugfix: Restore old behavior of `snapshots --latest <n>` without `--group-by`
Restic 0.19.0 accidentally changed the behavior of `snapshots --latest <n>`
to no longer group snapshots by host and paths by default.
The `snapshots --latest <n>` command now again uses the old behavior of
grouping by host and paths when `--group-by` is not specified. However, when
specifying `--group-by` the output is still grouped as requested, as in restic
0.19.0.
https://github.com/restic/restic/issues/21869
https://github.com/restic/restic/pull/21875
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Remove read-only files via the SFTP backend on Windows servers
Since restic 0.19.0, repository files on the SFTP backend are marked
read-only after save. On Windows SFTP servers, removing them failed
with a permission error. The SFTP backend now clears the read-only flag
before removing the file.
https://github.com/restic/restic/issues/21895
https://github.com/restic/restic/pull/21897
-8
View File
@@ -1,8 +0,0 @@
Bugfix: Make `backup` respect excludes for duplicate directory entries
Since restic 0.19.0, backing up a directory with duplicate directory
entries always produced "Warning: at least one source file could not be
read", even when those files were excluded. This has now been fixed.
https://github.com/restic/restic/issues/21899
https://github.com/restic/restic/pull/21900
-12
View File
@@ -1,12 +0,0 @@
Bugfix: Prevent mounting over the repository directory
Using a local repository directory as the `mount` target — or a path
that contains it, or that it contains — caused the FUSE server to
read its own backend files through the new mount, deadlocking the
kernel and requiring a long reboot to recover.
Restic now resolves both paths and refuses any such overlap with a
clear error before mounting.
https://github.com/restic/restic/issues/5234
https://github.com/restic/restic/pull/5348
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Skip inaccessible `backup` source paths
The `backup` command only skipped source paths that did not exist. A path that
could not be accessed for another reason, such as a malformed path on Windows,
was kept and produced an empty snapshot. Restic now skips any such path and
aborts if none remain.
https://github.com/restic/restic/issues/5667
https://github.com/restic/restic/pull/21852
-10
View File
@@ -1,10 +0,0 @@
Bugfix: Update `mount` latest symlink after snapshot reload
When `restic mount` was kept running while new snapshots were
created, the new snapshots appeared in the mountpoint, but the `latest`
symlink could still point to the previously latest snapshot. Restic now
invalidates the cached snapshot directory entries after a snapshot reload so
that `latest` points to the newest snapshot.
https://github.com/restic/restic/issues/5722
https://github.com/restic/restic/pull/21873
-9
View File
@@ -1,9 +0,0 @@
Bugfix: Show timezone location in `snapshots` output
With restic 0.19.0, the `snapshots` command printed the current timezone when
listing snapshots. However, that timezone label might change during the year,
for example with daylight saving time. Restic now prints a more consistent and
shorter version of the text.
https://github.com/restic/restic/pull/21876
https://forum.restic.net/t/possible-bug-in-timezone-naming/10867
-7
View File
@@ -1,7 +0,0 @@
Bugfix: Prevent crash in mountpoint validation if mountpoint is inaccessible
Since restic 0.19.0, the `mount` command validates a mountpoint before loading
the repository. If restic was unable to stat the mountpoint, this would result
in a crash. This has now been fixed to correctly return an error instead.
https://github.com/restic/restic/pull/21879
+5 -11
View File
@@ -1,20 +1,14 @@
# The first line must start with Bugfix:, Enhancement: or Change:, # The first line must start with Bugfix:, Enhancement: or Change:,
# including the colon. 'Change:' is for breaking changes only. # including the colon. Use present tense and the imperative mood. Remove
# Documentation-only changes do not get a changelog entry. # lines starting with '#' from this template.
# Include the affected command in the summary if relevant. Enhancement: Allow custom bar in the foo command
#
# Use present tense and the imperative mood. Remove lines starting
# with '#' from this template.
Enhancement: Allow custom bar in the `foo` command
# Describe the problem in the past tense, the new behavior in the present # Describe the problem in the past tense, the new behavior in the present
# tense. Mention the affected commands, backends, operating systems, etc. # tense. Mention the affected commands, backends, operating systems, etc.
# If the problem description just says that a feature was missing, then # If the problem description just says that a feature was missing, then
# only explain the new behavior. Aim for a short and concise description. # only explain the new behavior.
# Focus on user-facing behavior, not the implementation.
# Use "Restic now ..." instead of "We have changed ...". # Use "Restic now ..." instead of "We have changed ...".
#
# Focus on user-facing behavior, not the implementation. The description should
# be understandable for a regular user without knowledge of the implementation.
Restic foo always used the system-wide bar when deciding how to frob an Restic foo always used the system-wide bar when deciding how to frob an
item in the `baz` backend. It now permits selecting the bar with `--bar` item in the `baz` backend. It now permits selecting the bar with `--bar`
@@ -1,8 +1,7 @@
Enhancement: Support zstd compression levels `fastest` and `better` Enhancement: Added support for zstd compression levels `fastest` and `better`
Restic now supports the zstd compression modes `fastest` and `better`. Set the Restic now supports the zstd compression modes `fastest` and `better`. Set the
environment variable `RESTIC_COMPRESSION` to `fastest` or `better`, or pass the environment variable `RESTIC_COMPRESSION` to `fastest` or `better` to use these
same values with the `--compression` option. compression levels. This can also be set with the `--compression` flag.
https://github.com/restic/restic/issues/4728 https://github.com/restic/restic/issues/4728
https://github.com/restic/restic/pull/5321
+14
View File
@@ -0,0 +1,14 @@
Enhancement: Include repository id in filesystem name used by `mount`
The filesystem created by restic's `mount` command now includes the repository
id in the filesystem name. The repository id is printed by restic when opening
a repository or can be looked up using `restic cat config`.
```
[restic-user@hostname restic]$ df ./test-mount/
Filesystem 1K-blocks Used Available Use% Mounted on
restic:d3b07384d1 0 0 0 - /mnt/my-restic-repo
```
https://github.com/restic/restic/issues/4868
https://github.com/restic/restic/pull/5243
+8
View File
@@ -0,0 +1,8 @@
Bugfix: forget command returns exit code 3 on partial removal of snapshots
The `forget` command now returns exit code 3 when it fails to remove one or
more snapshots. Previously, it returned exit code 0, which could lead to
confusion if the command was used in a script.
https://github.com/restic/restic/issues/5233
https://github.com/restic/restic/pull/5322
@@ -1,14 +1,14 @@
Bugfix: Correctly handle `backup --stdin-filename` with directory paths Bugfix: Correctly handle `backup --stdin-filename` with directories
In restic 0.18.0, the `backup` command failed if a filename that includes In restic 0.18.0, the `backup` command failed if a filename that includes
at least a directory was passed to `--stdin-filename`. For example, a least a directory was passed to `--stdin-filename`. For example,
`--stdin-filename /foo/bar` resulted in the following error: `--stdin-filename /foo/bar` resulted in the following error:
``` ```
Fatal: unable to save snapshot: open /foo: no such file or directory Fatal: unable to save snapshot: open /foo: no such file or directory
``` ```
This has now been fixed. This has been fixed now.
https://github.com/restic/restic/issues/5324 https://github.com/restic/restic/issues/5324
https://github.com/restic/restic/pull/5356 https://github.com/restic/restic/pull/5356
@@ -1,7 +1,7 @@
Bugfix: Accept `RESTIC_HOST` environment variable in `forget` command Bugfix: Correctly handle `RESTIC_HOST` in `forget` command
The `forget` command did not use the host name from the `RESTIC_HOST` The `forget` command did not use the host name from the `RESTIC_HOST`
environment variable when filtering snapshots. This has now been fixed. environment variable. This has been fixed.
https://github.com/restic/restic/issues/5325 https://github.com/restic/restic/issues/5325
https://github.com/restic/restic/pull/5327 https://github.com/restic/restic/pull/5327
@@ -1,6 +1,6 @@
Bugfix: Ignore "chmod not supported" errors when writing files Bugfix: Ignore "chmod not supported" errors when writing files
Restic 0.18.0 introduced a bug that caused `chmod xxx: operation not supported` Restic 0.18.0 introduced a bug that caused "chmod xxx: operation not supported"
errors to appear when writing to a local file repository that did not support errors to appear when writing to a local file repository that did not support
chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors. chmod (like CIFS or WebDAV mounted via FUSE). Restic now ignores those errors.
+7
View File
@@ -0,0 +1,7 @@
Bugfix: Ignore EOPNOTSUPP as an error for xattr
Restic 0.18.0 added xattr support for NetBSD 10+, but not all NetBSD
filesystems support xattrs. Other BSD systems can likewise return
EOPNOTSUPP, so restic now simply ignores EOPNOTSUPP errors for xattrs.
https://github.com/restic/restic/issues/5344
+12
View File
@@ -0,0 +1,12 @@
Bugfix: Allow use of rclone/sftp backend when running restic in background
When starting restic in the background, this could result in unexpected behavior
when using the rclone or sftp backend.
For example running `restic -r rclone:./example --insecure-no-password init &`
could cause the calling `bash` shell to exit unexpectedly.
This has been fixed.
https://github.com/restic/restic/issues/5354
https://github.com/restic/restic/pull/5358
+8
View File
@@ -0,0 +1,8 @@
Bugfix: do not retry if rest-server runs out of space
Rest-server return error `507 Insufficient Storage` if no more storage
capacity is available at the server. Restic now no longer retries uploads
in this case.
https://github.com/restic/restic/issues/5429
https://github.com/restic/restic/pull/5452
@@ -1,7 +1,8 @@
Bugfix: Fix rare crash if directory is removed during backup Bugfix: Fix rare crash if directory is removed during backup
In restic 0.18.0, the `backup` command could crash if a directory was removed In restic 0.18.0, the `backup` command could crash if a directory is removed
between reading its metadata and listing its directory content. This has now inbetween reading its metadata and listing its directory content.
been fixed.
This has been fixed.
https://github.com/restic/restic/pull/5421 https://github.com/restic/restic/pull/5421
+7 -10
View File
@@ -2,8 +2,6 @@ package main
import ( import (
"context" "context"
"fmt"
"io"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
@@ -11,27 +9,26 @@ import (
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
) )
func createGlobalContext(stderr io.Writer) context.Context { func createGlobalContext() context.Context {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
ch := make(chan os.Signal, 1) ch := make(chan os.Signal, 1)
go cleanupHandler(ch, cancel, stderr) go cleanupHandler(ch, cancel)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
return ctx return ctx
} }
// cleanupHandler handles the SIGINT and SIGTERM signals. // cleanupHandler handles the SIGINT and SIGTERM signals.
func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc, stderr io.Writer) { func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
s := <-c s := <-c
debug.Log("signal %v received, cleaning up", s) debug.Log("signal %v received, cleaning up", s)
// ignore error as there's no good way to handle it Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
_, _ = fmt.Fprintf(stderr, "\rsignal %v received, cleaning up \n", s)
if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" { if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
_, _ = stderr.Write([]byte("\n--- STACKTRACE START ---\n\n")) _, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
_, _ = stderr.Write([]byte(debug.DumpStacktrace())) _, _ = os.Stderr.WriteString(debug.DumpStacktrace())
_, _ = stderr.Write([]byte("\n--- STACKTRACE END ---\n")) _, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
} }
cancel() cancel()
+84 -98
View File
@@ -19,20 +19,19 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/textfile" "github.com/restic/restic/internal/textfile"
"github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/backup" "github.com/restic/restic/internal/ui/backup"
"github.com/restic/restic/internal/ui/termstatus"
) )
func newBackupCommand(globalOptions *global.Options) *cobra.Command { func newBackupCommand() *cobra.Command {
var opts BackupOptions var opts BackupOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
@@ -52,28 +51,22 @@ Exit status is 10 if the repository does not exist.
Exit status is 11 if the repository is already locked. Exit status is 11 if the repository is already locked.
Exit status is 12 if the password is incorrect. Exit status is 12 if the password is incorrect.
`, `,
PreRunE: func(_ *cobra.Command, _ []string) error { PreRun: func(_ *cobra.Command, _ []string) {
if envVal := os.Getenv("RESTIC_READ_CONCURRENCY"); envVal != "" && !opts.readConcurrencyFlag.Changed {
n, err := strconv.ParseUint(envVal, 10, 32)
if err != nil {
return errors.Fatalf("invalid value for RESTIC_READ_CONCURRENCY %q: %v", envVal, err)
}
opts.ReadConcurrency = uint(n)
}
if opts.Host == "" { if opts.Host == "" {
hostname, err := os.Hostname() hostname, err := os.Hostname()
if err != nil { if err != nil {
debug.Log("os.Hostname() returned err: %v", err) debug.Log("os.Hostname() returned err: %v", err)
return nil return
} }
opts.Host = hostname opts.Host = hostname
} }
return nil
}, },
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runBackup(cmd.Context(), opts, *globalOptions, globalOptions.Term, args) term, cancel := setupTermstatus()
defer cancel()
return runBackup(cmd.Context(), opts, globalOptions, term, args)
}, },
} }
@@ -86,7 +79,7 @@ type BackupOptions struct {
filter.ExcludePatternOptions filter.ExcludePatternOptions
Parent string Parent string
GroupBy data.SnapshotGroupByOptions GroupBy restic.SnapshotGroupByOptions
Force bool Force bool
ExcludeOtherFS bool ExcludeOtherFS bool
ExcludeIfPresent []string ExcludeIfPresent []string
@@ -96,7 +89,7 @@ type BackupOptions struct {
Stdin bool Stdin bool
StdinFilename string StdinFilename string
StdinCommand bool StdinCommand bool
Tags data.TagLists Tags restic.TagLists
Host string Host string
FilesFrom []string FilesFrom []string
FilesFromVerbatim []string FilesFromVerbatim []string
@@ -110,13 +103,11 @@ type BackupOptions struct {
ReadConcurrency uint ReadConcurrency uint
NoScan bool NoScan bool
SkipIfUnchanged bool SkipIfUnchanged bool
readConcurrencyFlag *pflag.Flag
} }
func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) { func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) {
f.StringVar(&opts.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)") f.StringVar(&opts.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true} opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
f.BoolVarP(&opts.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`) f.BoolVarP(&opts.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
@@ -149,13 +140,13 @@ func (opts *BackupOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.NoScan, "no-scan", false, "do not run scanner to estimate size of backup") f.BoolVar(&opts.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
f.BoolVar(&opts.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)") f.BoolVar(&opts.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
} f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive Files On-Demand)")
if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
f.BoolVar(&opts.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive, iCloud drive, …)")
} }
f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot") f.BoolVar(&opts.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
opts.readConcurrencyFlag = f.Lookup("read-concurrency") // parse read concurrency from env, on error the default value will be used
readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
opts.ReadConcurrency = uint(readConcurrency)
// parse host from env, if not exists or empty the default value will be used // parse host from env, if not exists or empty the default value will be used
if host := os.Getenv("RESTIC_HOST"); host != "" { if host := os.Getenv("RESTIC_HOST"); host != "" {
@@ -168,20 +159,13 @@ var backupFSTestHook func(fs fs.FS) fs.FS
// ErrInvalidSourceData is used to report an incomplete backup // ErrInvalidSourceData is used to report an incomplete backup
var ErrInvalidSourceData = errors.New("at least one source file could not be read") var ErrInvalidSourceData = errors.New("at least one source file could not be read")
// ErrNoSourceData is used to report that no source data was found // filterExisting returns a slice of all existing items, or an error if no
var ErrNoSourceData = errors.Fatal("all source directories/files do not exist") // items exist at all.
func filterExisting(items []string) (result []string, err error) {
// filterExisting returns the items that exist and can be accessed. It returns
// ErrNoSourceData if none remain, or ErrInvalidSourceData if some were skipped.
func filterExisting(items []string, warnf func(msg string, args ...interface{})) (result []string, err error) {
for _, item := range items { for _, item := range items {
_, err := fs.Lstat(item) _, err := fs.Lstat(item)
if err != nil { if errors.Is(err, os.ErrNotExist) {
if errors.Is(err, os.ErrNotExist) { Warnf("%v does not exist, skipping\n", item)
warnf("%v does not exist, skipping\n", item)
} else {
warnf("%v cannot be accessed, skipping\n", item)
}
continue continue
} }
@@ -189,12 +173,10 @@ func filterExisting(items []string, warnf func(msg string, args ...interface{}))
} }
if len(result) == 0 { if len(result) == 0 {
return nil, ErrNoSourceData return nil, errors.Fatal("all source directories/files do not exist")
} else if len(result) < len(items) {
return result, ErrInvalidSourceData
} }
return result, nil return
} }
// readLines reads all lines from the named file and returns them as a // readLines reads all lines from the named file and returns them as a
@@ -203,7 +185,7 @@ func filterExisting(items []string, warnf func(msg string, args ...interface{}))
// If filename is empty, readPatternsFromFile returns an empty slice. // If filename is empty, readPatternsFromFile returns an empty slice.
// If filename is a dash (-), readPatternsFromFile will read the lines from the // If filename is a dash (-), readPatternsFromFile will read the lines from the
// standard input. // standard input.
func readLines(filename string, stdin io.ReadCloser) ([]string, error) { func readLines(filename string) ([]string, error) {
if filename == "" { if filename == "" {
return nil, nil return nil, nil
} }
@@ -214,7 +196,7 @@ func readLines(filename string, stdin io.ReadCloser) ([]string, error) {
) )
if filename == "-" { if filename == "-" {
data, err = io.ReadAll(stdin) data, err = io.ReadAll(os.Stdin)
} else { } else {
data, err = textfile.Read(filename) data, err = textfile.Read(filename)
} }
@@ -239,8 +221,8 @@ func readLines(filename string, stdin io.ReadCloser) ([]string, error) {
// readFilenamesFromFileRaw reads a list of filenames from the given file, // readFilenamesFromFileRaw reads a list of filenames from the given file,
// or stdin if filename is "-". Each filename is terminated by a zero byte, // or stdin if filename is "-". Each filename is terminated by a zero byte,
// which is stripped off. // which is stripped off.
func readFilenamesFromFileRaw(filename string, stdin io.ReadCloser) (names []string, err error) { func readFilenamesFromFileRaw(filename string) (names []string, err error) {
f := stdin f := os.Stdin
if filename != "-" { if filename != "-" {
if f, err = os.Open(filename); err != nil { if f, err = os.Open(filename); err != nil {
return nil, err return nil, err
@@ -289,8 +271,8 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
} }
// Check returns an error when an invalid combination of options was set. // Check returns an error when an invalid combination of options was set.
func (opts BackupOptions) Check(gopts global.Options, args []string) error { func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
if gopts.Password == "" && !gopts.InsecureNoPassword { if gopts.password == "" && !gopts.InsecureNoPassword {
if opts.Stdin { if opts.Stdin {
return errors.Fatal("cannot read both password and data from stdin") return errors.Fatal("cannot read both password and data from stdin")
} }
@@ -324,7 +306,7 @@ func (opts BackupOptions) Check(gopts global.Options, args []string) error {
// collectRejectByNameFuncs returns a list of all functions which may reject data // collectRejectByNameFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path only // from being saved in a snapshot based on path only
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, warnf func(msg string, args ...interface{})) (fs []archiver.RejectByNameFunc, err error) { func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) {
// exclude restic cache // exclude restic cache
if repo.Cache() != nil { if repo.Cache() != nil {
f, err := rejectResticCache(repo) f, err := rejectResticCache(repo)
@@ -335,7 +317,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, w
fs = append(fs, f) fs = append(fs, f)
} }
fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(warnf) fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -348,7 +330,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, w
// collectRejectFuncs returns a list of all functions which may reject data // collectRejectFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path and file info // from being saved in a snapshot based on path and file info
func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf func(msg string, args ...interface{})) (funcs []archiver.RejectFunc, err error) { func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS) (funcs []archiver.RejectFunc, err error) {
// allowed devices // allowed devices
if opts.ExcludeOtherFS && !opts.Stdin && !opts.StdinCommand { if opts.ExcludeOtherFS && !opts.Stdin && !opts.StdinCommand {
f, err := archiver.RejectByDevice(targets, fs) f, err := archiver.RejectByDevice(targets, fs)
@@ -372,7 +354,10 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu
} }
if opts.ExcludeCloudFiles && !opts.Stdin && !opts.StdinCommand { if opts.ExcludeCloudFiles && !opts.Stdin && !opts.StdinCommand {
f, err := archiver.RejectCloudFiles(warnf) if runtime.GOOS != "windows" {
return nil, errors.Fatalf("exclude-cloud-files is only supported on Windows")
}
f, err := archiver.RejectCloudFiles(Warnf)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -384,7 +369,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu
} }
for _, spec := range opts.ExcludeIfPresent { for _, spec := range opts.ExcludeIfPresent {
f, err := archiver.RejectIfPresent(spec, warnf) f, err := archiver.RejectIfPresent(spec, Warnf)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -396,13 +381,13 @@ func collectRejectFuncs(opts BackupOptions, targets []string, fs fs.FS, warnf fu
} }
// collectTargets returns a list of target files/dirs from several sources. // collectTargets returns a list of target files/dirs from several sources.
func collectTargets(opts BackupOptions, args []string, warnf func(msg string, args ...interface{}), stdin io.ReadCloser) (targets []string, err error) { func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
if opts.Stdin || opts.StdinCommand { if opts.Stdin || opts.StdinCommand {
return nil, nil return nil, nil
} }
for _, file := range opts.FilesFrom { for _, file := range opts.FilesFrom {
fromfile, err := readLines(file, stdin) fromfile, err := readLines(file)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -420,14 +405,14 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
return nil, fmt.Errorf("pattern: %s: %w", line, err) return nil, fmt.Errorf("pattern: %s: %w", line, err)
} }
if len(expanded) == 0 { if len(expanded) == 0 {
warnf("pattern %q does not match any files, skipping\n", line) Warnf("pattern %q does not match any files, skipping\n", line)
} }
targets = append(targets, expanded...) targets = append(targets, expanded...)
} }
} }
for _, file := range opts.FilesFromVerbatim { for _, file := range opts.FilesFromVerbatim {
fromfile, err := readLines(file, stdin) fromfile, err := readLines(file)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -440,7 +425,7 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
} }
for _, file := range opts.FilesFromRaw { for _, file := range opts.FilesFromRaw {
fromfile, err := readFilenamesFromFileRaw(file, stdin) fromfile, err := readFilenamesFromFileRaw(file)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -454,12 +439,17 @@ func collectTargets(opts BackupOptions, args []string, warnf func(msg string, ar
return nil, errors.Fatal("nothing to backup, please specify source files/dirs") return nil, errors.Fatal("nothing to backup, please specify source files/dirs")
} }
return filterExisting(targets, warnf) targets, err = filterExisting(targets)
if err != nil {
return nil, err
}
return targets, nil
} }
// parent returns the ID of the parent snapshot. If there is none, nil is // parent returns the ID of the parent snapshot. If there is none, nil is
// returned. // returned.
func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*data.Snapshot, error) { func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
if opts.Force { if opts.Force {
return nil, nil return nil, nil
} }
@@ -468,7 +458,7 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o
if snName == "" { if snName == "" {
snName = "latest" snName = "latest"
} }
f := data.SnapshotFilter{TimestampLimit: timeStampLimit} f := restic.SnapshotFilter{TimestampLimit: timeStampLimit}
if opts.GroupBy.Host { if opts.GroupBy.Host {
f.Hosts = []string{opts.Host} f.Hosts = []string{opts.Host}
} }
@@ -476,29 +466,23 @@ func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, o
f.Paths = targets f.Paths = targets
} }
if opts.GroupBy.Tag { if opts.GroupBy.Tag {
f.Tags = []data.TagList{opts.Tags.Flatten()} f.Tags = []restic.TagList{opts.Tags.Flatten()}
} }
sn, _, err := f.FindLatest(ctx, repo, repo, snName) sn, _, err := f.FindLatest(ctx, repo, repo, snName)
// Snapshot not found is ok if no explicit parent was set // Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, data.ErrNoSnapshotFound) { if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil err = nil
} }
return sn, err return sn, err
} }
func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, term ui.Terminal, args []string) error { func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
var vsscfg fs.VSSConfig var vsscfg fs.VSSConfig
var err error var err error
var printer backup.ProgressPrinter
if gopts.JSON {
printer = backup.NewJSONProgress(term, gopts.Verbosity)
} else {
printer = backup.NewTextProgress(term, gopts.Verbosity)
}
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
if vsscfg, err = fs.ParseVSSConfig(gopts.Extended); err != nil { if vsscfg, err = fs.ParseVSSConfig(gopts.extended); err != nil {
return err return err
} }
} }
@@ -508,46 +492,47 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
return err return err
} }
success := true targets, err := collectTargets(opts, args)
targets, err := collectTargets(opts, args, printer.E, term.InputRaw())
if err != nil { if err != nil {
if errors.Is(err, ErrInvalidSourceData) { return err
success = false
} else {
return err
}
} }
timeStamp := time.Now() timeStamp := time.Now()
backupStart := timeStamp backupStart := timeStamp
if opts.TimeStamp != "" { if opts.TimeStamp != "" {
timeStamp, err = time.ParseInLocation(global.TimeFormat, opts.TimeStamp, time.Local) timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
if err != nil { if err != nil {
return errors.Fatalf("error in time option: %v", err) return errors.Fatalf("error in time option: %v\n", err)
} }
} }
if gopts.Verbosity >= 2 && !gopts.JSON { if gopts.verbosity >= 2 && !gopts.JSON {
printer.P("open repository") Verbosef("open repository\n")
} }
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun, printer) ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun)
if err != nil { if err != nil {
return err return err
} }
defer unlock() defer unlock()
progressReporter := backup.NewProgress(printer, var progressPrinter backup.ProgressPrinter
ui.CalculateProgressInterval(!gopts.Quiet, gopts.JSON, term.CanUpdateStatus())) if gopts.JSON {
progressPrinter = backup.NewJSONProgress(term, gopts.verbosity)
} else {
progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
}
progressReporter := backup.NewProgress(progressPrinter,
calculateProgressInterval(!gopts.Quiet, gopts.JSON))
defer progressReporter.Done() defer progressReporter.Done()
// rejectByNameFuncs collect functions that can reject items from the backup based on path only // rejectByNameFuncs collect functions that can reject items from the backup based on path only
rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo, printer.E) rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
if err != nil { if err != nil {
return err return err
} }
var parentSnapshot *data.Snapshot var parentSnapshot *restic.Snapshot
if !opts.Stdin { if !opts.Stdin {
parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp) parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
if err != nil { if err != nil {
@@ -556,18 +541,19 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
if !gopts.JSON { if !gopts.JSON {
if parentSnapshot != nil { if parentSnapshot != nil {
printer.P("using parent snapshot %v\n", parentSnapshot.ID().Str()) progressPrinter.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
} else { } else {
printer.P("no parent snapshot found, will read all files\n") progressPrinter.P("no parent snapshot found, will read all files\n")
} }
} }
} }
if !gopts.JSON { if !gopts.JSON {
printer.V("load index files") progressPrinter.V("load index files")
} }
err = repo.LoadIndex(ctx, printer) bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
err = repo.LoadIndex(ctx, bar)
if err != nil { if err != nil {
return err return err
} }
@@ -584,7 +570,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
messageHandler := func(msg string, args ...interface{}) { messageHandler := func(msg string, args ...interface{}) {
if !gopts.JSON { if !gopts.JSON {
printer.P(msg, args...) progressPrinter.P(msg, args...)
} }
} }
@@ -595,12 +581,12 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
if opts.Stdin || opts.StdinCommand { if opts.Stdin || opts.StdinCommand {
if !gopts.JSON { if !gopts.JSON {
printer.V("read data from stdin") progressPrinter.V("read data from stdin")
} }
filename := path.Join("/", opts.StdinFilename) filename := path.Join("/", opts.StdinFilename)
source := term.InputRaw() var source io.ReadCloser = os.Stdin
if opts.StdinCommand { if opts.StdinCommand {
source, err = fs.NewCommandReader(ctx, args, printer.E) source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
if err != nil { if err != nil {
return err return err
} }
@@ -620,7 +606,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
} }
// rejectFuncs collect functions that can reject items from the backup based on path and file info // rejectFuncs collect functions that can reject items from the backup based on path and file info
rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS, printer.E) rejectFuncs, err := collectRejectFuncs(opts, targets, targetFS)
if err != nil { if err != nil {
return err return err
} }
@@ -636,11 +622,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
sc := archiver.NewScanner(targetFS) sc := archiver.NewScanner(targetFS)
sc.SelectByName = selectByNameFilter sc.SelectByName = selectByNameFilter
sc.Select = selectFilter sc.Select = selectFilter
sc.Error = printer.ScannerError sc.Error = progressPrinter.ScannerError
sc.Result = progressReporter.ReportTotal sc.Result = progressReporter.ReportTotal
if !gopts.JSON { if !gopts.JSON {
printer.V("start scan on %v", targets) progressPrinter.V("start scan on %v", targets)
} }
wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
} }
@@ -649,7 +635,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
arch.SelectByName = selectByNameFilter arch.SelectByName = selectByNameFilter
arch.Select = selectFilter arch.Select = selectFilter
arch.WithAtime = opts.WithAtime arch.WithAtime = opts.WithAtime
success := true
arch.Error = func(item string, err error) error { arch.Error = func(item string, err error) error {
success = false success = false
reterr := progressReporter.Error(item, err) reterr := progressReporter.Error(item, err)
@@ -680,12 +666,12 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts global.Options, te
Time: timeStamp, Time: timeStamp,
Hostname: opts.Host, Hostname: opts.Host,
ParentSnapshot: parentSnapshot, ParentSnapshot: parentSnapshot,
ProgramVersion: "restic " + global.Version, ProgramVersion: "restic " + version,
SkipIfUnchanged: opts.SkipIfUnchanged, SkipIfUnchanged: opts.SkipIfUnchanged,
} }
if !gopts.JSON { if !gopts.JSON {
printer.V("start backup on %v", targets) progressPrinter.V("start backup on %v", targets)
} }
_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts) _, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
+44 -48
View File
@@ -3,34 +3,33 @@ package main
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
) )
func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) error { func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
return withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error { return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
t.Logf("backing up %v in %v", target, dir) t.Logf("backing up %v in %v", target, dir)
if dir != "" { if dir != "" {
cleanup := rtest.Chdir(t, dir) cleanup := rtest.Chdir(t, dir)
defer cleanup() defer cleanup()
} }
opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true} opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
return runBackup(ctx, opts, gopts, gopts.Term, target) return runBackup(ctx, opts, gopts, term, target)
}) })
} }
func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts global.Options) { func testRunBackup(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) {
err := testRunBackupAssumeFailure(t, dir, target, opts, gopts) err := testRunBackupAssumeFailure(t, dir, target, opts, gopts)
rtest.Assert(t, err == nil, "Error while backing up: %v", err) rtest.Assert(t, err == nil, "Error while backing up: %v", err)
} }
@@ -57,13 +56,13 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
testListSnapshots(t, env.gopts, 1) testListSnapshots(t, env.gopts, 1)
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
stat1 := dirStats(t, env.repo) stat1 := dirStats(env.repo)
// second backup, implicit incremental // second backup, implicit incremental
testRunBackup(t, "", []string{env.testdata}, opts, env.gopts) testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
snapshotIDs := testListSnapshots(t, env.gopts, 2) snapshotIDs := testListSnapshots(t, env.gopts, 2)
stat2 := dirStats(t, env.repo) stat2 := dirStats(env.repo)
if stat2.size > stat1.size+stat1.size/10 { if stat2.size > stat1.size+stat1.size/10 {
t.Error("repository size has grown by more than 10 percent") t.Error("repository size has grown by more than 10 percent")
} }
@@ -75,7 +74,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
testRunBackup(t, "", []string{env.testdata}, opts, env.gopts) testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
snapshotIDs = testListSnapshots(t, env.gopts, 3) snapshotIDs = testListSnapshots(t, env.gopts, 3)
stat3 := dirStats(t, env.repo) stat3 := dirStats(env.repo)
if stat3.size > stat1.size+stat1.size/10 { if stat3.size > stat1.size+stat1.size/10 {
t.Error("repository size has grown by more than 10 percent") t.Error("repository size has grown by more than 10 percent")
} }
@@ -86,7 +85,7 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir) t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
testRunRestore(t, env.gopts, restoredir, snapshotID.String()+":"+toPathInSnapshot(filepath.Dir(env.testdata))) testRunRestore(t, env.gopts, restoredir, snapshotID.String()+":"+toPathInSnapshot(filepath.Dir(env.testdata)))
diff := directoriesContentsDiff(t, env.testdata, filepath.Join(restoredir, "testdata")) diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
rtest.Assert(t, diff == "", "directories are not equal: %v", diff) rtest.Assert(t, diff == "", "directories are not equal: %v", diff)
} }
@@ -219,41 +218,41 @@ func TestDryRunBackup(t *testing.T) {
// dry run before first backup // dry run before first backup
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts) testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDs := testListSnapshots(t, env.gopts, 0) snapshotIDs := testListSnapshots(t, env.gopts, 0)
packIDs := testRunList(t, env.gopts, "packs") packIDs := testRunList(t, "packs", env.gopts)
rtest.Assert(t, len(packIDs) == 0, rtest.Assert(t, len(packIDs) == 0,
"expected no data, got %v", snapshotIDs) "expected no data, got %v", snapshotIDs)
indexIDs := testRunList(t, env.gopts, "index") indexIDs := testRunList(t, "index", env.gopts)
rtest.Assert(t, len(indexIDs) == 0, rtest.Assert(t, len(indexIDs) == 0,
"expected no index, got %v", snapshotIDs) "expected no index, got %v", snapshotIDs)
// first backup // first backup
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshotIDs = testListSnapshots(t, env.gopts, 1) snapshotIDs = testListSnapshots(t, env.gopts, 1)
packIDs = testRunList(t, env.gopts, "packs") packIDs = testRunList(t, "packs", env.gopts)
indexIDs = testRunList(t, env.gopts, "index") indexIDs = testRunList(t, "index", env.gopts)
// dry run between backups // dry run between backups
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts) testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDsAfter := testListSnapshots(t, env.gopts, 1) snapshotIDsAfter := testListSnapshots(t, env.gopts, 1)
rtest.Equals(t, snapshotIDs, snapshotIDsAfter) rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
dataIDsAfter := testRunList(t, env.gopts, "packs") dataIDsAfter := testRunList(t, "packs", env.gopts)
rtest.Equals(t, packIDs, dataIDsAfter) rtest.Equals(t, packIDs, dataIDsAfter)
indexIDsAfter := testRunList(t, env.gopts, "index") indexIDsAfter := testRunList(t, "index", env.gopts)
rtest.Equals(t, indexIDs, indexIDsAfter) rtest.Equals(t, indexIDs, indexIDsAfter)
// second backup, implicit incremental // second backup, implicit incremental
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshotIDs = testListSnapshots(t, env.gopts, 2) snapshotIDs = testListSnapshots(t, env.gopts, 2)
packIDs = testRunList(t, env.gopts, "packs") packIDs = testRunList(t, "packs", env.gopts)
indexIDs = testRunList(t, env.gopts, "index") indexIDs = testRunList(t, "index", env.gopts)
// another dry run // another dry run
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts) testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDsAfter = testListSnapshots(t, env.gopts, 2) snapshotIDsAfter = testListSnapshots(t, env.gopts, 2)
rtest.Equals(t, snapshotIDs, snapshotIDsAfter) rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
dataIDsAfter = testRunList(t, env.gopts, "packs") dataIDsAfter = testRunList(t, "packs", env.gopts)
rtest.Equals(t, packIDs, dataIDsAfter) rtest.Equals(t, packIDs, dataIDsAfter)
indexIDsAfter = testRunList(t, env.gopts, "index") indexIDsAfter = testRunList(t, "index", env.gopts)
rtest.Equals(t, indexIDs, indexIDsAfter) rtest.Equals(t, indexIDs, indexIDsAfter)
} }
@@ -263,27 +262,22 @@ func TestBackupNonExistingFile(t *testing.T) {
testSetupBackupData(t, env) testSetupBackupData(t, env)
p := filepath.Join(env.testdata, "0", "0", "9") _ = withRestoreGlobalOptions(func() error {
dirs := []string{ globalOptions.stderr = io.Discard
filepath.Join(p, "0"),
filepath.Join(p, "1"),
filepath.Join(p, "nonexisting"),
filepath.Join(p, "5"),
}
opts := BackupOptions{} p := filepath.Join(env.testdata, "0", "0", "9")
dirs := []string{
filepath.Join(p, "0"),
filepath.Join(p, "1"),
filepath.Join(p, "nonexisting"),
filepath.Join(p, "5"),
}
// mix of existing and non-existing files opts := BackupOptions{}
err := testRunBackupAssumeFailure(t, "", dirs, opts, env.gopts)
rtest.Assert(t, err != nil, "expected error for non-existing file") testRunBackup(t, "", dirs, opts, env.gopts)
rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "expected ErrInvalidSourceData; got %v", err) return nil
// only non-existing file })
dirs = []string{
filepath.Join(p, "nonexisting"),
}
err = testRunBackupAssumeFailure(t, "", dirs, opts, env.gopts)
rtest.Assert(t, err != nil, "expected error for non-existing file")
rtest.Assert(t, errors.Is(err, ErrNoSourceData), "expected ErrNoSourceData; got %v", err)
} }
func TestBackupSelfHealing(t *testing.T) { func TestBackupSelfHealing(t *testing.T) {
@@ -444,13 +438,13 @@ func TestIncrementalBackup(t *testing.T) {
testRunBackup(t, "", []string{datadir}, opts, env.gopts) testRunBackup(t, "", []string{datadir}, opts, env.gopts)
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
stat1 := dirStats(t, env.repo) stat1 := dirStats(env.repo)
rtest.OK(t, appendRandomData(testfile, incrementalSecondWrite)) rtest.OK(t, appendRandomData(testfile, incrementalSecondWrite))
testRunBackup(t, "", []string{datadir}, opts, env.gopts) testRunBackup(t, "", []string{datadir}, opts, env.gopts)
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
stat2 := dirStats(t, env.repo) stat2 := dirStats(env.repo)
if stat2.size-stat1.size > incrementalFirstWrite { if stat2.size-stat1.size > incrementalFirstWrite {
t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite) t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
} }
@@ -460,13 +454,14 @@ func TestIncrementalBackup(t *testing.T) {
testRunBackup(t, "", []string{datadir}, opts, env.gopts) testRunBackup(t, "", []string{datadir}, opts, env.gopts)
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
stat3 := dirStats(t, env.repo) stat3 := dirStats(env.repo)
if stat3.size-stat2.size > incrementalFirstWrite { if stat3.size-stat2.size > incrementalFirstWrite {
t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite) t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
} }
t.Logf("repository grown by %d bytes", stat3.size-stat2.size) t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
} }
// nolint: staticcheck // false positive nil pointer dereference check
func TestBackupTags(t *testing.T) { func TestBackupTags(t *testing.T) {
env, cleanup := withTestEnvironment(t) env, cleanup := withTestEnvironment(t)
defer cleanup() defer cleanup()
@@ -486,7 +481,7 @@ func TestBackupTags(t *testing.T) {
"expected no tags, got %v", newest.Tags) "expected no tags, got %v", newest.Tags)
parent := newest parent := newest
opts.Tags = data.TagLists{[]string{"NL"}} opts.Tags = restic.TagLists{[]string{"NL"}}
testRunBackup(t, "", []string{env.testdata}, opts, env.gopts) testRunBackup(t, "", []string{env.testdata}, opts, env.gopts)
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
newest, _ = testRunSnapshots(t, env.gopts) newest, _ = testRunSnapshots(t, env.gopts)
@@ -502,6 +497,7 @@ func TestBackupTags(t *testing.T) {
"expected parent to be %v, got %v", parent.ID, newest.Parent) "expected parent to be %v, got %v", parent.ID, newest.Parent)
} }
// nolint: staticcheck // false positive nil pointer dereference check
func TestBackupProgramVersion(t *testing.T) { func TestBackupProgramVersion(t *testing.T) {
env, cleanup := withTestEnvironment(t) env, cleanup := withTestEnvironment(t)
defer cleanup() defer cleanup()
@@ -513,7 +509,7 @@ func TestBackupProgramVersion(t *testing.T) {
if newest == nil { if newest == nil {
t.Fatal("expected a backup, got nil") t.Fatal("expected a backup, got nil")
} }
resticVersion := "restic " + global.Version resticVersion := "restic " + version
rtest.Assert(t, newest.ProgramVersion == resticVersion, rtest.Assert(t, newest.ProgramVersion == resticVersion,
"expected %v, got %v", resticVersion, newest.ProgramVersion) "expected %v, got %v", resticVersion, newest.ProgramVersion)
} }
@@ -571,7 +567,7 @@ func TestHardLink(t *testing.T) {
restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i)) restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir) t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
testRunRestore(t, env.gopts, restoredir, snapshotID.String()) testRunRestore(t, env.gopts, restoredir, snapshotID.String())
diff := directoriesContentsDiff(t, env.testdata, filepath.Join(restoredir, "testdata")) diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, "testdata"))
rtest.Assert(t, diff == "", "directories are not equal %v", diff) rtest.Assert(t, diff == "", "directories are not equal %v", diff)
linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata")) linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
@@ -707,7 +703,7 @@ func TestBackupEmptyPassword(t *testing.T) {
env, cleanup := withTestEnvironment(t) env, cleanup := withTestEnvironment(t)
defer cleanup() defer cleanup()
env.gopts.Password = "" env.gopts.password = ""
env.gopts.InsecureNoPassword = true env.gopts.InsecureNoPassword = true
testSetupBackupData(t, env) testSetupBackupData(t, env)
+1 -27
View File
@@ -10,7 +10,6 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/errors"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
@@ -68,35 +67,10 @@ func TestCollectTargets(t *testing.T) {
FilesFromRaw: []string{f3.Name()}, FilesFromRaw: []string{f3.Name()},
} }
targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")}, t.Logf, nil) targets, err := collectTargets(opts, []string{filepath.Join(dir, "cmdline arg")})
rtest.OK(t, err) rtest.OK(t, err)
sort.Strings(targets) sort.Strings(targets)
rtest.Equals(t, expect, targets) rtest.Equals(t, expect, targets)
_, err = collectTargets(opts, []string{filepath.Join(dir, "cmdline arg"), filepath.Join(dir, "non-existing-file")}, t.Logf, nil)
rtest.Assert(t, err == ErrInvalidSourceData, "expected error when not all targets exist")
}
func TestFilterExistingUnreadable(t *testing.T) {
dir := rtest.TempDir(t)
existing := filepath.Join(dir, "existing")
rtest.OK(t, os.Mkdir(existing, 0755))
file := filepath.Join(dir, "file")
rtest.OK(t, os.WriteFile(file, []byte("x"), 0600))
// Regression test for #5667. A target whose Lstat fails with an error other
// than ErrNotExist must be skipped (ENOTDIR on unix, NUL byte everywhere).
for _, unreadable := range []string{filepath.Join(file, "child"), "invalid\x00path"} {
result, err := filterExisting([]string{unreadable}, t.Logf)
rtest.Assert(t, errors.Is(err, ErrNoSourceData), "input %q: expected ErrNoSourceData; got %v", unreadable, err)
rtest.Assert(t, len(result) == 0, "input %q: expected no targets; got %v", unreadable, result)
result, err = filterExisting([]string{existing, unreadable}, t.Logf)
rtest.Assert(t, errors.Is(err, ErrInvalidSourceData), "input %q: expected ErrInvalidSourceData; got %v", unreadable, err)
rtest.Equals(t, []string{existing}, result)
}
} }
func TestReadFilenamesRaw(t *testing.T) { func TestReadFilenamesRaw(t *testing.T) {
+9 -12
View File
@@ -10,14 +10,13 @@ import (
"github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/backend/cache"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/table" "github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
func newCacheCommand(globalOptions *global.Options) *cobra.Command { func newCacheCommand() *cobra.Command {
var opts CacheOptions var opts CacheOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
@@ -35,7 +34,7 @@ Exit status is 1 if there was any error.
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, args []string) error {
return runCache(opts, *globalOptions, args, globalOptions.Term) return runCache(opts, globalOptions, args)
}, },
} }
@@ -56,9 +55,7 @@ func (opts *CacheOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.NoSize, "no-size", false, "do not output the size of the cache directories") f.BoolVar(&opts.NoSize, "no-size", false, "do not output the size of the cache directories")
} }
func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Terminal) error { func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
if len(args) > 0 { if len(args) > 0 {
return errors.Fatal("the cache command expects no arguments, only options - please see `restic help cache` for usage and flags") return errors.Fatal("the cache command expects no arguments, only options - please see `restic help cache` for usage and flags")
} }
@@ -86,17 +83,17 @@ func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Te
} }
if len(oldDirs) == 0 { if len(oldDirs) == 0 {
printer.P("no old cache dirs found") Verbosef("no old cache dirs found\n")
return nil return nil
} }
printer.P("remove %d old cache directories", len(oldDirs)) Verbosef("remove %d old cache directories\n", len(oldDirs))
for _, item := range oldDirs { for _, item := range oldDirs {
dir := filepath.Join(cachedir, item.Name()) dir := filepath.Join(cachedir, item.Name())
err = os.RemoveAll(dir) err = os.RemoveAll(dir)
if err != nil { if err != nil {
printer.E("unable to remove %v: %v", dir, err) Warnf("unable to remove %v: %v\n", dir, err)
} }
} }
@@ -126,7 +123,7 @@ func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Te
} }
if len(dirs) == 0 { if len(dirs) == 0 {
printer.S("no cache dirs found, basedir is %v", cachedir) Printf("no cache dirs found, basedir is %v\n", cachedir)
return nil return nil
} }
@@ -162,8 +159,8 @@ func runCache(opts CacheOptions, gopts global.Options, args []string, term ui.Te
}) })
} }
_ = tab.Write(gopts.Term.OutputWriter()) _ = tab.Write(globalOptions.stdout)
printer.S("%d cache dirs in %s", len(dirs), cachedir) Printf("%d cache dirs in %s\n", len(dirs), cachedir)
return nil return nil
} }
+24 -27
View File
@@ -7,17 +7,14 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
) )
var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"} var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"}
func newCatCommand(globalOptions *global.Options) *cobra.Command { func newCatCommand() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]", Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]",
Short: "Print internal objects to stdout", Short: "Print internal objects to stdout",
@@ -36,7 +33,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runCat(cmd.Context(), *globalOptions, args, globalOptions.Term) return runCat(cmd.Context(), globalOptions, args)
}, },
ValidArgs: catAllowedCmds, ValidArgs: catAllowedCmds,
} }
@@ -66,14 +63,12 @@ func validateCatArgs(args []string) error {
return nil return nil
} }
func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error { func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
if err := validateCatArgs(args); err != nil { if err := validateCatArgs(args); err != nil {
return err return err
} }
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil { if err != nil {
return err return err
} }
@@ -85,7 +80,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" { if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" && tpe != "tree" {
id, err = restic.ParseID(args[1]) id, err = restic.ParseID(args[1])
if err != nil { if err != nil {
return errors.Fatalf("unable to parse ID: %v", err) return errors.Fatalf("unable to parse ID: %v\n", err)
} }
} }
@@ -96,7 +91,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err return err
} }
printer.S(string(buf)) Println(string(buf))
return nil return nil
case "index": case "index":
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id) buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
@@ -104,12 +99,12 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err return err
} }
printer.S(string(buf)) Println(string(buf))
return nil return nil
case "snapshot": case "snapshot":
sn, _, err := data.FindSnapshot(ctx, repo, repo, args[1]) sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil { if err != nil {
return errors.Fatalf("could not find snapshot: %v", err) return errors.Fatalf("could not find snapshot: %v\n", err)
} }
buf, err := json.MarshalIndent(sn, "", " ") buf, err := json.MarshalIndent(sn, "", " ")
@@ -117,7 +112,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err return err
} }
printer.S(string(buf)) Println(string(buf))
return nil return nil
case "key": case "key":
key, err := repository.LoadKey(ctx, repo, id) key, err := repository.LoadKey(ctx, repo, id)
@@ -130,7 +125,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err return err
} }
printer.S(string(buf)) Println(string(buf))
return nil return nil
case "masterkey": case "masterkey":
buf, err := json.MarshalIndent(repo.Key(), "", " ") buf, err := json.MarshalIndent(repo.Key(), "", " ")
@@ -138,7 +133,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err return err
} }
printer.S(string(buf)) Println(string(buf))
return nil return nil
case "lock": case "lock":
lock, err := restic.LoadLock(ctx, repo, id) lock, err := restic.LoadLock(ctx, repo, id)
@@ -151,7 +146,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err return err
} }
printer.S(string(buf)) Println(string(buf))
return nil return nil
case "pack": case "pack":
@@ -163,14 +158,15 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
hash := restic.Hash(buf) hash := restic.Hash(buf)
if !hash.Equal(id) { if !hash.Equal(id) {
printer.E("Warning: hash of data does not match ID, want\n %v\ngot:\n %v", id.String(), hash.String()) Warnf("Warning: hash of data does not match ID, want\n %v\ngot:\n %v\n", id.String(), hash.String())
} }
_, err = term.OutputRaw().Write(buf) _, err = globalOptions.stdout.Write(buf)
return err return err
case "blob": case "blob":
err = repo.LoadIndex(ctx, printer) bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil { if err != nil {
return err return err
} }
@@ -185,24 +181,25 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
return err return err
} }
_, err = term.OutputRaw().Write(buf) _, err = globalOptions.stdout.Write(buf)
return err return err
} }
return errors.Fatal("blob not found") return errors.Fatal("blob not found")
case "tree": case "tree":
sn, subfolder, err := data.FindSnapshot(ctx, repo, repo, args[1]) sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
if err != nil { if err != nil {
return errors.Fatalf("could not find snapshot: %v", err) return errors.Fatalf("could not find snapshot: %v\n", err)
} }
err = repo.LoadIndex(ctx, printer) bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil { if err != nil {
return err return err
} }
sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil { if err != nil {
return err return err
} }
@@ -211,7 +208,7 @@ func runCat(ctx context.Context, gopts global.Options, args []string, term ui.Te
if err != nil { if err != nil {
return err return err
} }
_, err = term.OutputRaw().Write(buf) _, err = globalOptions.stdout.Write(buf)
return err return err
default: default:
+70 -104
View File
@@ -15,30 +15,25 @@ import (
"github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/backend/cache"
"github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress" "github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
) )
func newCheckCommand(globalOptions *global.Options) *cobra.Command { func newCheckCommand() *cobra.Command {
var opts CheckOptions var opts CheckOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "check [flags]", Use: "check [flags]",
Short: "Check the repository for errors", Short: "Check the repository for errors",
Long: ` Long: `
The "check" command tests the repository for errors and reports any errors it The "check" command tests the repository for errors and reports any errors it
finds. finds. It can also be used to read all data and therefore simulate a restore.
By default, check verifies the structural consistency and integrity of By default, the "check" command will always load all data directly from the
snapshots, trees and pack files. To also verify the integrity of the actual repository and not use a local cache.
backed-up data, use the --read-data or --read-data-subset flags.
By default, check creates a new temporary cache directory to verify data.
To reuse the existing cache, use the --with-cache flag.
EXIT STATUS EXIT STATUS
=========== ===========
@@ -52,13 +47,14 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter) term, cancel := setupTermstatus()
summary, err := runCheck(cmd.Context(), opts, *globalOptions, args, globalOptions.Term) defer cancel()
summary, err := runCheck(cmd.Context(), opts, globalOptions, args, term)
if globalOptions.JSON { if globalOptions.JSON {
if err != nil && summary.NumErrors == 0 { if err != nil && summary.NumErrors == 0 {
summary.NumErrors = 1 summary.NumErrors = 1
} }
globalOptions.Term.Print(ui.ToJSONString(summary)) term.Print(ui.ToJSONString(summary))
} }
return err return err
}, },
@@ -77,7 +73,6 @@ type CheckOptions struct {
ReadDataSubset string ReadDataSubset string
CheckUnused bool CheckUnused bool
WithCache bool WithCache bool
data.SnapshotFilter
} }
func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) { func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
@@ -91,7 +86,6 @@ func (opts *CheckOptions) AddFlags(f *pflag.FlagSet) {
panic(err) panic(err)
} }
f.BoolVar(&opts.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository") f.BoolVar(&opts.WithCache, "with-cache", false, "use existing cache, only read uncached data from repository")
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
} }
func checkFlags(opts CheckOptions) error { func checkFlags(opts CheckOptions) error {
@@ -179,7 +173,7 @@ func parsePercentage(s string) (float64, error) {
// - if the user explicitly requested --no-cache, we don't use any cache // - if the user explicitly requested --no-cache, we don't use any cache
// - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check // - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
// - by default, we use a cache in a temporary directory that is deleted after the check // - by default, we use a cache in a temporary directory that is deleted after the check
func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progress.Printer) (cleanup func()) { func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress.Printer) (cleanup func()) {
cleanup = func() {} cleanup = func() {}
if opts.WithCache { if opts.WithCache {
// use the default cache, no setup needed // use the default cache, no setup needed
@@ -200,7 +194,7 @@ func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progres
// use a cache in a temporary directory // use a cache in a temporary directory
err := os.MkdirAll(cachedir, 0755) err := os.MkdirAll(cachedir, 0755)
if err != nil { if err != nil {
printer.E("unable to create cache directory %s, disabling cache: %v", cachedir, err) Warnf("unable to create cache directory %s, disabling cache: %v\n", cachedir, err)
gopts.NoCache = true gopts.NoCache = true
return cleanup return cleanup
} }
@@ -226,12 +220,15 @@ func prepareCheckCache(opts CheckOptions, gopts *global.Options, printer progres
return cleanup return cleanup
} }
func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args []string, term ui.Terminal) (checkSummary, error) { func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) (checkSummary, error) {
summary := checkSummary{MessageType: "summary"} summary := checkSummary{MessageType: "summary"}
if len(args) != 0 {
return summary, errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
}
var printer progress.Printer var printer progress.Printer
if !gopts.JSON { if !gopts.JSON {
printer = ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term) printer = newTerminalProgressPrinter(gopts.verbosity, term)
} else { } else {
printer = newJSONErrorPrinter(term) printer = newJSONErrorPrinter(term)
} }
@@ -242,38 +239,32 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
if !gopts.NoLock { if !gopts.NoLock {
printer.P("create exclusive lock for repository\n") printer.P("create exclusive lock for repository\n")
} }
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock, printer) ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
if err != nil { if err != nil {
return summary, err return summary, err
} }
defer unlock() defer unlock()
chkr := checker.New(repo, opts.CheckUnused) chkr := checker.New(repo, opts.CheckUnused)
err = chkr.LoadSnapshots(ctx, &opts.SnapshotFilter, args) err = chkr.LoadSnapshots(ctx)
if err != nil { if err != nil {
return summary, err return summary, err
} }
printer.P("load indexes\n") printer.P("load indexes\n")
hints, errs := chkr.LoadIndex(ctx, printer) bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
hints, errs := chkr.LoadIndex(ctx, bar)
if ctx.Err() != nil { if ctx.Err() != nil {
return summary, ctx.Err() return summary, ctx.Err()
} }
errorsFound := false errorsFound := false
salvagePacks := restic.NewIDSet()
for _, hint := range hints { for _, hint := range hints {
switch hint := hint.(type) { switch hint.(type) {
case *repository.ErrIncompletePackEntry: case *checker.ErrDuplicatePacks:
printer.E("%s", hint.Error())
salvagePacks.Insert(hint.PackID)
errorsFound = true
summary.NumErrors++
case *repository.ErrDuplicatePacks:
printer.S("%s", hint.Error()) printer.S("%s", hint.Error())
summary.HintRepairIndex = true summary.HintRepairIndex = true
case *repository.ErrMixedPack: case *checker.ErrMixedPack:
printer.S("%s", hint.Error()) printer.S("%s", hint.Error())
summary.HintPrune = true summary.HintPrune = true
default: default:
@@ -302,12 +293,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
orphanedPacks := 0 orphanedPacks := 0
errChan := make(chan error) errChan := make(chan error)
salvagePacks := restic.NewIDSet()
printer.P("check all packs\n") printer.P("check all packs\n")
go chkr.Packs(ctx, errChan) go chkr.Packs(ctx, errChan)
for err := range errChan { for err := range errChan {
var packErr *repository.PackError var packErr *checker.PackError
if errors.As(err, &packErr) { if errors.As(err, &packErr) {
if packErr.Orphaned { if packErr.Orphaned {
orphanedPacks++ orphanedPacks++
@@ -371,7 +363,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
return summary, ctx.Err() return summary, ctx.Err()
} }
// the following block only used for tests
if opts.CheckUnused { if opts.CheckUnused {
unused, err := chkr.UnusedBlobs(ctx) unused, err := chkr.UnusedBlobs(ctx)
if err != nil { if err != nil {
@@ -383,16 +374,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
} }
} }
readDataFilter, err := buildPacksFilter(opts, printer, chkr.IsFiltered()) doReadData := func(packs map[restic.ID]int64) {
if err != nil {
return summary, err
}
if readDataFilter != nil {
p := printer.NewCounter("packs") p := printer.NewCounter("packs")
p.SetMax(uint64(len(packs)))
errChan := make(chan error) errChan := make(chan error)
go chkr.ReadPacks(ctx, readDataFilter, p, errChan) go chkr.ReadPacks(ctx, packs, p, errChan)
for err := range errChan { for err := range errChan {
errorsFound = true errorsFound = true
@@ -405,6 +392,48 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
p.Done() p.Done()
} }
switch {
case opts.ReadData:
printer.P("read all data\n")
doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
case opts.ReadDataSubset != "":
var packs map[restic.ID]int64
dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
if err == nil {
bucket := dataSubset[0]
totalBuckets := dataSubset[1]
packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
packCount := uint64(len(packs))
printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
percentage, err := parsePercentage(opts.ReadDataSubset)
if err == nil {
packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
printer.P("read %.1f%% of data packs\n", percentage)
}
} else {
repoSize := int64(0)
allPacks := chkr.GetPacks()
for _, size := range allPacks {
repoSize += size
}
if repoSize == 0 {
return summary, errors.Fatal("Cannot read from a repository having size 0")
}
subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
if subsetSize > repoSize {
subsetSize = repoSize
}
packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
percentage := float64(subsetSize) / float64(repoSize) * 100.0
printer.P("read %d bytes (%.1f%%) of data packs\n", subsetSize, percentage)
}
if packs == nil {
return summary, errors.Fatal("internal error: failed to select packs to check")
}
doReadData(packs)
}
if len(salvagePacks) > 0 { if len(salvagePacks) > 0 {
printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
for id := range salvagePacks { for id := range salvagePacks {
@@ -428,64 +457,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts global.Options, args
return summary, nil return summary, nil
} }
func buildPacksFilter(opts CheckOptions, printer progress.Printer,
filteredStatus bool) (func(packs map[restic.ID]int64) map[restic.ID]int64, error) {
typeData := ""
if filteredStatus {
typeData = "filtered "
}
switch {
case opts.ReadData:
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
printer.P("read all %sdata", typeData)
return packs
}, nil
case opts.ReadDataSubset != "":
dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
if err == nil {
bucket := dataSubset[0]
totalBuckets := dataSubset[1]
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
packCount := uint64(len(packs))
packs = selectPacksByBucket(packs, bucket, totalBuckets)
printer.P("read group #%d of %d %sdata packs (out of total %d packs in %d groups", bucket, len(packs), typeData, packCount, totalBuckets)
return packs
}, nil
} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
percentage, err := parsePercentage(opts.ReadDataSubset)
if err != nil {
return nil, err
}
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
printer.P("read %.1f%% of %spackfiles", percentage, typeData)
return selectRandomPacksByPercentage(packs, percentage)
}, nil
}
repoSize := int64(0)
return func(packs map[restic.ID]int64) map[restic.ID]int64 {
for _, size := range packs {
repoSize += size
}
subsetSize, _ := ui.ParseBytes(opts.ReadDataSubset)
if subsetSize > repoSize {
subsetSize = repoSize
}
if repoSize > 0 {
packs = selectRandomPacksByFileSize(packs, subsetSize, repoSize)
}
percentage := float64(subsetSize) / float64(repoSize) * 100.0
if repoSize == 0 {
percentage = 100
}
printer.P("read %d bytes (%.1f%%) of %sdata packs\n", subsetSize, percentage, typeData)
return packs
}, nil
}
return nil, nil
}
// selectPacksByBucket selects subsets of packs by ranges of buckets. // selectPacksByBucket selects subsets of packs by ranges of buckets.
func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 { func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 {
packs := make(map[restic.ID]int64) packs := make(map[restic.ID]int64)
@@ -557,10 +528,6 @@ func (*jsonErrorPrinter) NewCounter(_ string) *progress.Counter {
return nil return nil
} }
func (*jsonErrorPrinter) NewCounterTerminalOnly(_ string) *progress.Counter {
return nil
}
func (p *jsonErrorPrinter) E(msg string, args ...interface{}) { func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
status := checkError{ status := checkError{
MessageType: "error", MessageType: "error",
@@ -570,6 +537,5 @@ func (p *jsonErrorPrinter) E(msg string, args ...interface{}) {
} }
func (*jsonErrorPrinter) S(_ string, _ ...interface{}) {} func (*jsonErrorPrinter) S(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) P(_ string, _ ...interface{}) {} func (*jsonErrorPrinter) P(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) PT(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) V(_ string, _ ...interface{}) {} func (*jsonErrorPrinter) V(_ string, _ ...interface{}) {}
func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {} func (*jsonErrorPrinter) VV(_ string, _ ...interface{}) {}
+11 -73
View File
@@ -1,101 +1,39 @@
package main package main
import ( import (
"bytes"
"context" "context"
"strings"
"testing" "testing"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
) )
func testRunCheck(t testing.TB, gopts global.Options) { func testRunCheck(t testing.TB, gopts GlobalOptions) {
t.Helper() t.Helper()
output, err := testRunCheckOutput(t, gopts, true) output, err := testRunCheckOutput(gopts, true)
if err != nil { if err != nil {
t.Error(output) t.Error(output)
t.Fatalf("unexpected error: %+v", err) t.Fatalf("unexpected error: %+v", err)
} }
} }
func testRunCheckMustFail(t testing.TB, gopts global.Options) { func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
t.Helper() t.Helper()
_, err := testRunCheckOutput(t, gopts, false) _, err := testRunCheckOutput(gopts, false)
rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository") rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository")
} }
func testRunCheckOutput(t testing.TB, gopts global.Options, checkUnused bool) (string, error) { func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error { buf := bytes.NewBuffer(nil)
gopts.stdout = buf
err := withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
opts := CheckOptions{ opts := CheckOptions{
ReadData: true, ReadData: true,
CheckUnused: checkUnused, CheckUnused: checkUnused,
} }
_, err := runCheck(context.TODO(), opts, gopts, nil, gopts.Term) _, err := runCheck(context.TODO(), opts, gopts, nil, term)
return err return err
}) })
return buf.String(), err return buf.String(), err
} }
func testRunCheckOutputWithOpts(t testing.TB, gopts global.Options, opts CheckOptions, args []string) (string, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error {
gopts.Verbosity = 2
_, err := runCheck(context.TODO(), opts, gopts, args, gopts.Term)
return err
})
return buf.String(), err
}
func TestCheckWithSnaphotFilter(t *testing.T) {
testCases := []struct {
opts CheckOptions
args []string
expectedOutput string
}{
{ // full --read-data, all snapshots
CheckOptions{ReadData: true},
nil,
"4 / 4 packs",
},
{ // full --read-data, all snapshots
CheckOptions{ReadData: true},
nil,
"2 / 2 snapshots",
},
{ // full --read-data, latest snapshot
CheckOptions{ReadData: true},
[]string{"latest"},
"2 / 2 packs",
},
{ // full --read-data, latest snapshot
CheckOptions{ReadData: true},
[]string{"latest"},
"1 / 1 snapshots",
},
{ // --read-data-subset, latest snapshot
CheckOptions{ReadDataSubset: "1%"},
[]string{"latest"},
"1 / 1 packs",
},
{ // --read-data-subset, latest snapshot
CheckOptions{ReadDataSubset: "1%"},
[]string{"latest"},
"filtered",
},
}
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
opts := BackupOptions{}
testRunBackup(t, env.testdata+"/0", []string{"for_cmd_ls"}, opts, env.gopts)
testRunBackup(t, env.testdata+"/0", []string{"0/9"}, opts, env.gopts)
for _, testCase := range testCases {
output, err := testRunCheckOutputWithOpts(t, env.gopts, testCase.opts, testCase.args)
rtest.OK(t, err)
hasOutput := strings.Contains(output, testCase.expectedOutput)
rtest.Assert(t, hasOutput, `expected to find substring %q, but did not find it`, testCase.expectedOutput)
}
}
+2 -3
View File
@@ -9,7 +9,6 @@ import (
"testing" "testing"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/progress" "github.com/restic/restic/internal/ui/progress"
@@ -203,7 +202,7 @@ func TestPrepareCheckCache(t *testing.T) {
err := os.Remove(tmpDirBase) err := os.Remove(tmpDirBase)
rtest.OK(t, err) rtest.OK(t, err)
} }
gopts := global.Options{CacheDir: tmpDirBase} gopts := GlobalOptions{CacheDir: tmpDirBase}
cleanup := prepareCheckCache(testCase.opts, &gopts, &progress.NoopPrinter{}) cleanup := prepareCheckCache(testCase.opts, &gopts, &progress.NoopPrinter{})
files, err := os.ReadDir(tmpDirBase) files, err := os.ReadDir(tmpDirBase)
rtest.OK(t, err) rtest.OK(t, err)
@@ -233,7 +232,7 @@ func TestPrepareCheckCache(t *testing.T) {
} }
func TestPrepareDefaultCheckCache(t *testing.T) { func TestPrepareDefaultCheckCache(t *testing.T) {
gopts := global.Options{CacheDir: ""} gopts := GlobalOptions{CacheDir: ""}
cleanup := prepareCheckCache(CheckOptions{}, &gopts, &progress.NoopPrinter{}) cleanup := prepareCheckCache(CheckOptions{}, &gopts, &progress.NoopPrinter{})
_, err := os.ReadDir(gopts.CacheDir) _, err := os.ReadDir(gopts.CacheDir)
rtest.OK(t, err) rtest.OK(t, err)
+109 -197
View File
@@ -3,24 +3,18 @@ package main
import ( import (
"context" "context"
"fmt" "fmt"
"iter"
"sync"
"time"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui" "golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/ui/progress"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
func newCopyCommand(globalOptions *global.Options) *cobra.Command { func newCopyCommand() *cobra.Command {
var opts CopyOptions var opts CopyOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "copy [flags] [snapshotID ...]", Use: "copy [flags] [snapshotID ...]",
@@ -52,8 +46,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter) return runCopy(cmd.Context(), opts, globalOptions, args)
return runCopy(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
}, },
} }
@@ -63,51 +56,17 @@ Exit status is 12 if the password is incorrect.
// CopyOptions bundles all options for the copy command. // CopyOptions bundles all options for the copy command.
type CopyOptions struct { type CopyOptions struct {
global.SecondaryRepoOptions secondaryRepoOptions
data.SnapshotFilter restic.SnapshotFilter
} }
func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) { func (opts *CopyOptions) AddFlags(f *pflag.FlagSet) {
opts.SecondaryRepoOptions.AddFlags(f, "destination", "to copy snapshots from") opts.secondaryRepoOptions.AddFlags(f, "destination", "to copy snapshots from")
initMultiSnapshotFilter(f, &opts.SnapshotFilter, true) initMultiSnapshotFilter(f, &opts.SnapshotFilter, true)
} }
// collectAllSnapshots: select all snapshot trees to be copied func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
func collectAllSnapshots(ctx context.Context, opts CopyOptions, secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
srcSnapshotLister restic.Lister, srcRepo restic.Repository,
dstSnapshotByOriginal map[restic.ID][]*data.Snapshot, args []string, printer progress.Printer,
) iter.Seq[*data.Snapshot] {
return func(yield func(*data.Snapshot) bool) {
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args, printer) {
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
if sn.Original != nil {
srcOriginal = *sn.Original
}
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
printer.V("\n%v", sn)
printer.V("skipping source snapshot %s, was already copied to snapshot %s", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
}
if isCopy {
continue
}
}
if !yield(sn) {
return
}
}
}
}
func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args []string, term ui.Terminal) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
secondaryGopts, isFromRepo, err := opts.SecondaryRepoOptions.FillGlobalOpts(ctx, gopts, "destination")
if err != nil { if err != nil {
return err return err
} }
@@ -116,13 +75,13 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [
gopts, secondaryGopts = secondaryGopts, gopts gopts, secondaryGopts = secondaryGopts, gopts
} }
ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer) ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil { if err != nil {
return err return err
} }
defer unlock() defer unlock()
ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false, printer) ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false)
if err != nil { if err != nil {
return err return err
} }
@@ -139,16 +98,18 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [
} }
debug.Log("Loading source index") debug.Log("Loading source index")
if err := srcRepo.LoadIndex(ctx, printer); err != nil { bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err := srcRepo.LoadIndex(ctx, bar); err != nil {
return err return err
} }
bar = newIndexProgress(gopts.Quiet, gopts.JSON)
debug.Log("Loading destination index") debug.Log("Loading destination index")
if err := dstRepo.LoadIndex(ctx, printer); err != nil { if err := dstRepo.LoadIndex(ctx, bar); err != nil {
return err return err
} }
dstSnapshotByOriginal := make(map[restic.ID][]*data.Snapshot) dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil, printer) { for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) {
if sn.Original != nil && !sn.Original.IsNull() { if sn.Original != nil && !sn.Original.IsNull() {
dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn) dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
} }
@@ -159,16 +120,53 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts global.Options, args [
return ctx.Err() return ctx.Err()
} }
selectedSnapshots := collectAllSnapshots(ctx, opts, srcSnapshotLister, srcRepo, dstSnapshotByOriginal, args, printer) // remember already processed trees across all snapshots
visitedTrees := restic.NewIDSet()
if err := copyTreeBatched(ctx, srcRepo, dstRepo, selectedSnapshots, printer); err != nil { for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) {
return err // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
if sn.Original != nil {
srcOriginal = *sn.Original
}
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
Verboseff("\n%v\n", sn)
Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
}
if isCopy {
continue
}
}
Verbosef("\n%v\n", sn)
Verbosef(" copy started, this may take a while...\n")
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
return err
}
debug.Log("tree copied")
// save snapshot
sn.Parent = nil // Parent does not have relevance in the new repo.
// Use Original as a persistent snapshot ID
if sn.Original == nil {
sn.Original = sn.ID()
}
newID, err := restic.SaveSnapshot(ctx, dstRepo, sn)
if err != nil {
return err
}
Verbosef("snapshot %s saved\n", newID.Str())
} }
return ctx.Err() return ctx.Err()
} }
func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool { func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
// everything except Parent and Original must match // everything except Parent and Original must match
if !sna.Time.Equal(snb.Time) || !sna.Tree.Equal(*snb.Tree) || sna.Hostname != snb.Hostname || if !sna.Time.Equal(snb.Time) || !sna.Tree.Equal(*snb.Tree) || sna.Hostname != snb.Hostname ||
sna.Username != snb.Username || sna.UID != snb.UID || sna.GID != snb.GID || sna.Username != snb.Username || sna.UID != snb.UID || sna.GID != snb.GID ||
@@ -187,158 +185,72 @@ func similarSnapshots(sna *data.Snapshot, snb *data.Snapshot) bool {
return true return true
} }
// copyTreeBatched copies multiple snapshots in one go. Snapshots are written after
// data equivalent to at least 10 packfiles was written.
func copyTreeBatched(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
selectedSnapshots iter.Seq[*data.Snapshot], printer progress.Printer) error {
// remember already processed trees across all snapshots
visitedTrees := srcRepo.NewAssociatedBlobSet()
targetSize := uint64(dstRepo.PackSize()) * 100
minDuration := 1 * time.Minute
// use pull-based iterator to allow iteration in multiple steps
next, stop := iter.Pull(selectedSnapshots)
defer stop()
for {
var batch []*data.Snapshot
batchSize := uint64(0)
startTime := time.Now()
// call WithBlobUploader() once and then loop over all selectedSnapshots
err := dstRepo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error {
for batchSize < targetSize || time.Since(startTime) < minDuration {
sn, ok := next()
if !ok {
break
}
batch = append(batch, sn)
printer.P("\n%v", sn)
printer.P(" copy started, this may take a while...")
sizeBlobs, err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, printer, uploader)
if err != nil {
return err
}
debug.Log("tree copied")
batchSize += sizeBlobs
}
return nil
})
if err != nil {
return err
}
// if no snapshots were processed in this batch, we're done
if len(batch) == 0 {
break
}
// add a newline to separate saved snapshot messages from the other messages
if len(batch) > 1 {
printer.P("")
}
// save all the snapshots
for _, sn := range batch {
err := copySaveSnapshot(ctx, sn, dstRepo, printer)
if err != nil {
return err
}
}
}
return nil
}
func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository, func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Repository,
visitedTrees restic.AssociatedBlobSet, rootTreeID restic.ID, printer progress.Printer, uploader restic.BlobSaverWithAsync) (uint64, error) { visitedTrees restic.IDSet, rootTreeID restic.ID, quiet bool) error {
copyBlobs := srcRepo.NewAssociatedBlobSet() wg, wgCtx := errgroup.WithContext(ctx)
treeStream := restic.StreamTrees(wgCtx, wg, srcRepo, restic.IDs{rootTreeID}, func(treeID restic.ID) bool {
visited := visitedTrees.Has(treeID)
visitedTrees.Insert(treeID)
return visited
}, nil)
copyBlobs := restic.NewBlobSet()
packList := restic.NewIDSet() packList := restic.NewIDSet()
var lock sync.Mutex
enqueue := func(h restic.BlobHandle) { enqueue := func(h restic.BlobHandle) {
lock.Lock() pb := srcRepo.LookupBlob(h.Type, h.ID)
defer lock.Unlock() copyBlobs.Insert(h)
if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok { for _, p := range pb {
pb := srcRepo.LookupBlob(h.Type, h.ID) packList.Insert(p.PackID)
copyBlobs.Insert(h)
for _, p := range pb {
packList.Insert(p.PackID)
}
} }
} }
err := data.StreamTrees(ctx, srcRepo, restic.IDs{rootTreeID}, nil, func(treeID restic.ID) bool { wg.Go(func() error {
handle := restic.BlobHandle{ID: treeID, Type: restic.TreeBlob} for tree := range treeStream {
visited := visitedTrees.Has(handle) if tree.Error != nil {
visitedTrees.Insert(handle) return fmt.Errorf("LoadTree(%v) returned error %v", tree.ID.Str(), tree.Error)
return visited
}, func(treeID restic.ID, err error, nodes data.TreeNodeIterator) error {
if err != nil {
return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
}
// copy raw tree bytes to avoid problems if the serialization changes
enqueue(restic.BlobHandle{ID: treeID, Type: restic.TreeBlob})
for item := range nodes {
if item.Error != nil {
return item.Error
} }
// Recursion into directories is handled by StreamTrees
// Copy the blobs for this file. // Do we already have this tree blob?
for _, blobID := range item.Node.Content { treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
enqueue(restic.BlobHandle{Type: restic.DataBlob, ID: blobID}) if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok {
// copy raw tree bytes to avoid problems if the serialization changes
enqueue(treeHandle)
}
for _, entry := range tree.Nodes {
// Recursion into directories is handled by StreamTrees
// Copy the blobs for this file.
for _, blobID := range entry.Content {
h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
enqueue(h)
}
}
} }
} }
return nil return nil
}) })
if err != nil { err := wg.Wait()
return 0, err
}
sizeBlobs := copyStats(srcRepo, copyBlobs, packList, printer)
bar := printer.NewCounter("packs copied")
err = repository.CopyBlobs(ctx, srcRepo, dstRepo, uploader, packList, copyBlobs, bar, printer.P)
if err != nil {
return 0, errors.Fatalf("%s", err)
}
return sizeBlobs, nil
}
// copyStats: print statistics for the blobs to be copied
func copyStats(srcRepo restic.Repository, copyBlobs restic.AssociatedBlobSet, packList restic.IDSet, printer progress.Printer) uint64 {
// count and size
countBlobs := 0
sizeBlobs := uint64(0)
for blob := range copyBlobs.Keys() {
for _, blob := range srcRepo.LookupBlob(blob.Type, blob.ID) {
countBlobs++
sizeBlobs += uint64(blob.Length)
break
}
}
printer.V(" copy %d blobs with disk size %s in %d packfiles\n",
countBlobs, ui.FormatBytes(uint64(sizeBlobs)), len(packList))
return sizeBlobs
}
func copySaveSnapshot(ctx context.Context, sn *data.Snapshot, dstRepo restic.Repository, printer progress.Printer) error {
sn.Parent = nil // Parent does not have relevance in the new repo.
// Use Original as a persistent snapshot ID
if sn.Original == nil {
sn.Original = sn.ID()
}
newID, err := data.SaveSnapshot(ctx, dstRepo, sn)
if err != nil { if err != nil {
return err return err
} }
printer.P("snapshot %s saved, copied from source snapshot %s", newID.Str(), sn.ID().Str())
bar := newProgressMax(!quiet, uint64(len(packList)), "packs copied")
_, err = repository.Repack(
ctx,
srcRepo,
dstRepo,
packList,
copyBlobs,
bar,
func(msg string, args ...interface{}) { fmt.Printf(msg+"\n", args...) },
)
bar.Done()
if err != nil {
return errors.Fatal(err.Error())
}
return nil return nil
} }
+9 -49
View File
@@ -6,28 +6,23 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
) )
func testRunCopy(t testing.TB, srcGopts global.Options, dstGopts global.Options) { func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
gopts := srcGopts gopts := srcGopts
gopts.Repo = dstGopts.Repo gopts.Repo = dstGopts.Repo
gopts.Password = dstGopts.Password gopts.password = dstGopts.password
gopts.InsecureNoPassword = dstGopts.InsecureNoPassword gopts.InsecureNoPassword = dstGopts.InsecureNoPassword
copyOpts := CopyOptions{ copyOpts := CopyOptions{
SecondaryRepoOptions: global.SecondaryRepoOptions{ secondaryRepoOptions: secondaryRepoOptions{
Repo: srcGopts.Repo, Repo: srcGopts.Repo,
Password: srcGopts.Password, password: srcGopts.password,
InsecureNoPassword: srcGopts.InsecureNoPassword, InsecureNoPassword: srcGopts.InsecureNoPassword,
}, },
} }
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error { rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
return runCopy(context.TODO(), copyOpts, gopts, nil, gopts.Term)
}))
} }
func TestCopy(t *testing.T) { func TestCopy(t *testing.T) {
@@ -50,8 +45,8 @@ func TestCopy(t *testing.T) {
copiedSnapshotIDs := testListSnapshots(t, env2.gopts, 3) copiedSnapshotIDs := testListSnapshots(t, env2.gopts, 3)
// Check that the copies size seems reasonable // Check that the copies size seems reasonable
stat := dirStats(t, env.repo) stat := dirStats(env.repo)
stat2 := dirStats(t, env2.repo) stat2 := dirStats(env2.repo)
sizeDiff := int64(stat.size) - int64(stat2.size) sizeDiff := int64(stat.size) - int64(stat2.size)
if sizeDiff < 0 { if sizeDiff < 0 {
sizeDiff = -sizeDiff sizeDiff = -sizeDiff
@@ -74,7 +69,7 @@ func TestCopy(t *testing.T) {
testRunRestore(t, env2.gopts, restoredir, snapshotID.String()) testRunRestore(t, env2.gopts, restoredir, snapshotID.String())
foundMatch := false foundMatch := false
for cmpdir := range origRestores { for cmpdir := range origRestores {
diff := directoriesContentsDiff(t, restoredir, cmpdir) diff := directoriesContentsDiff(restoredir, cmpdir)
if diff == "" { if diff == "" {
delete(origRestores, cmpdir) delete(origRestores, cmpdir)
foundMatch = true foundMatch = true
@@ -85,41 +80,6 @@ func TestCopy(t *testing.T) {
} }
rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots") rtest.Assert(t, len(origRestores) == 0, "found not copied snapshots")
// check that snapshots were properly batched while copying
_, _, countBlobs := testPackAndBlobCounts(t, env.gopts)
countTreePacksDst, countDataPacksDst, countBlobsDst := testPackAndBlobCounts(t, env2.gopts)
rtest.Equals(t, countBlobs, countBlobsDst, "expected blob count in both repos to be equal")
rtest.Equals(t, countTreePacksDst, 1, "expected 1 tree packfile")
rtest.Equals(t, countDataPacksDst, 1, "expected 1 data packfile")
}
func testPackAndBlobCounts(t testing.TB, gopts global.Options) (countTreePacks int, countDataPacks int, countBlobs int) {
rtest.OK(t, withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
rtest.OK(t, err)
defer unlock()
rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(context.TODO(), id, size)
rtest.OK(t, err)
rtest.Assert(t, len(blobs) > 0, "a packfile should contain at least one blob")
switch blobs[0].Type {
case restic.TreeBlob:
countTreePacks++
case restic.DataBlob:
countDataPacks++
}
countBlobs += len(blobs)
return nil
}))
return nil
}))
return countTreePacks, countDataPacks, countBlobs
} }
func TestCopyIncremental(t *testing.T) { func TestCopyIncremental(t *testing.T) {
@@ -182,7 +142,7 @@ func TestCopyToEmptyPassword(t *testing.T) {
defer cleanup() defer cleanup()
env2, cleanup2 := withTestEnvironment(t) env2, cleanup2 := withTestEnvironment(t)
defer cleanup2() defer cleanup2()
env2.gopts.Password = "" env2.gopts.password = ""
env2.gopts.InsecureNoPassword = true env2.gopts.InsecureNoPassword = true
testSetupBackupData(t, env) testSetupBackupData(t, env)
+91 -83
View File
@@ -1,4 +1,5 @@
//go:build debug //go:build debug
// +build debug
package main package main
@@ -11,6 +12,7 @@ import (
"io" "io"
"os" "os"
"runtime" "runtime"
"sort"
"sync" "sync"
"time" "time"
@@ -20,36 +22,32 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/repository/index"
"github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/repository/pack"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
) )
func registerDebugCommand(cmd *cobra.Command, globalOptions *global.Options) { func registerDebugCommand(cmd *cobra.Command) {
cmd.AddCommand( cmd.AddCommand(
newDebugCommand(globalOptions), newDebugCommand(),
) )
} }
func newDebugCommand(globalOptions *global.Options) *cobra.Command { func newDebugCommand() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "debug", Use: "debug",
Short: "Debug commands", Short: "Debug commands",
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
} }
cmd.AddCommand(newDebugDumpCommand(globalOptions)) cmd.AddCommand(newDebugDumpCommand())
cmd.AddCommand(newDebugExamineCommand(globalOptions)) cmd.AddCommand(newDebugExamineCommand())
return cmd return cmd
} }
func newDebugDumpCommand(globalOptions *global.Options) *cobra.Command { func newDebugDumpCommand() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "dump [indexes|snapshots|all|packs]", Use: "dump [indexes|snapshots|all|packs]",
Short: "Dump data structures", Short: "Dump data structures",
@@ -68,13 +66,13 @@ Exit status is 12 if the password is incorrect.
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runDebugDump(cmd.Context(), *globalOptions, args, globalOptions.Term) return runDebugDump(cmd.Context(), globalOptions, args)
}, },
} }
return cmd return cmd
} }
func newDebugExamineCommand(globalOptions *global.Options) *cobra.Command { func newDebugExamineCommand() *cobra.Command {
var opts DebugExamineOptions var opts DebugExamineOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
@@ -82,7 +80,7 @@ func newDebugExamineCommand(globalOptions *global.Options) *cobra.Command {
Short: "Examine a pack file", Short: "Examine a pack file",
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runDebugExamine(cmd.Context(), *globalOptions, opts, args, globalOptions.Term) return runDebugExamine(cmd.Context(), globalOptions, opts, args)
}, },
} }
@@ -115,7 +113,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
} }
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error { func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
return data.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *data.Snapshot, err error) error { return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
if err != nil { if err != nil {
return err return err
} }
@@ -143,13 +141,13 @@ type Blob struct {
Offset uint `json:"offset"` Offset uint `json:"offset"`
} }
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer, printer progress.Printer) error { func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
var m sync.Mutex var m sync.Mutex
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
blobs, _, err := repo.ListPack(ctx, id, size) blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil { if err != nil {
printer.E("error for pack %v: %v", id.Str(), err) Warnf("error for pack %v: %v\n", id.Str(), err)
return nil return nil
} }
@@ -172,9 +170,9 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer,
}) })
} }
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer, printer progress.Printer) error { func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error { return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, err error) error {
printer.S("index_id: %v", id) Printf("index_id: %v\n", id)
if err != nil { if err != nil {
return err return err
} }
@@ -183,14 +181,12 @@ func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Wr
}) })
} }
func runDebugDump(ctx context.Context, gopts global.Options, args []string, term ui.Terminal) error { func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
if len(args) != 1 { if len(args) != 1 {
return errors.Fatal("type not specified") return errors.Fatal("type not specified")
} }
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil { if err != nil {
return err return err
} }
@@ -200,20 +196,20 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
switch tpe { switch tpe {
case "indexes": case "indexes":
return dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) return dumpIndexes(ctx, repo, globalOptions.stdout)
case "snapshots": case "snapshots":
return debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter()) return debugPrintSnapshots(ctx, repo, globalOptions.stdout)
case "packs": case "packs":
return printPacks(ctx, repo, gopts.Term.OutputWriter(), printer) return printPacks(ctx, repo, globalOptions.stdout)
case "all": case "all":
printer.S("snapshots:") Printf("snapshots:\n")
err := debugPrintSnapshots(ctx, repo, gopts.Term.OutputWriter()) err := debugPrintSnapshots(ctx, repo, globalOptions.stdout)
if err != nil { if err != nil {
return err return err
} }
printer.S("indexes:") Printf("\nindexes:\n")
err = dumpIndexes(ctx, repo, gopts.Term.OutputWriter(), printer) err = dumpIndexes(ctx, repo, globalOptions.stdout)
if err != nil { if err != nil {
return err return err
} }
@@ -224,11 +220,11 @@ func runDebugDump(ctx context.Context, gopts global.Options, args []string, term
} }
} }
func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer progress.Printer) []byte { func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool) []byte {
if bytewise { if bytewise {
printer.S(" trying to repair blob by finding a broken byte") Printf(" trying to repair blob by finding a broken byte\n")
} else { } else {
printer.S(" trying to repair blob with single bit flip") Printf(" trying to repair blob with single bit flip\n")
} }
ch := make(chan int) ch := make(chan int)
@@ -238,7 +234,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
var found bool var found bool
workers := runtime.GOMAXPROCS(0) workers := runtime.GOMAXPROCS(0)
printer.S(" spinning up %d worker functions", runtime.GOMAXPROCS(0)) Printf(" spinning up %d worker functions\n", runtime.GOMAXPROCS(0))
for i := 0; i < workers; i++ { for i := 0; i < workers; i++ {
wg.Go(func() error { wg.Go(func() error {
// make a local copy of the buffer // make a local copy of the buffer
@@ -252,9 +248,9 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil) plaintext, err := key.Open(plaintext[:0], nonce, plaintext, nil)
if err == nil { if err == nil {
printer.S("") Printf("\n")
printer.S(" blob could be repaired by XORing byte %v with 0x%02x", idx, pattern) Printf(" blob could be repaired by XORing byte %v with 0x%02x\n", idx, pattern)
printer.S(" hash is %v", restic.Hash(plaintext)) Printf(" hash is %v\n", restic.Hash(plaintext))
close(done) close(done)
found = true found = true
fixed = plaintext fixed = plaintext
@@ -295,7 +291,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
select { select {
case ch <- i: case ch <- i:
case <-done: case <-done:
printer.S(" done after %v", time.Since(start)) Printf(" done after %v\n", time.Since(start))
return nil return nil
} }
@@ -305,7 +301,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
remaining := len(input) - i remaining := len(input) - i
eta := time.Duration(float64(remaining)/gps) * time.Second eta := time.Duration(float64(remaining)/gps) * time.Second
printer.S("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v", Printf("\r%d byte of %d done (%.2f%%), %.0f byte per second, ETA %v",
i, len(input), float32(i)/float32(len(input))*100, gps, eta) i, len(input), float32(i)/float32(len(input))*100, gps, eta)
info = time.Now() info = time.Now()
} }
@@ -318,7 +314,7 @@ func tryRepairWithBitflip(key *crypto.Key, input []byte, bytewise bool, printer
} }
if !found { if !found {
printer.S("\n blob could not be repaired") Printf("\n blob could not be repaired\n")
} }
return fixed return fixed
} }
@@ -339,7 +335,7 @@ func decryptUnsigned(k *crypto.Key, buf []byte) []byte {
return out return out
} }
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list restic.Blobs, printer progress.Printer) error { func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
dec, err := zstd.NewReader(nil) dec, err := zstd.NewReader(nil)
if err != nil { if err != nil {
panic(err) panic(err)
@@ -351,11 +347,17 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
return err return err
} }
err = repo.WithBlobUploader(ctx, func(ctx context.Context, uploader restic.BlobSaverWithAsync) error { wg, ctx := errgroup.WithContext(ctx)
if opts.ReuploadBlobs {
repo.StartPackUploader(ctx, wg)
}
wg.Go(func() error {
for _, blob := range list { for _, blob := range list {
printer.S(" loading blob %v at %v (length %v)", blob.ID, blob.Offset, blob.Length) Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
if int(blob.Offset+blob.Length) > len(pack) { if int(blob.Offset+blob.Length) > len(pack) {
printer.E("skipping truncated blob") Warnf("skipping truncated blob\n")
continue continue
} }
buf := pack[blob.Offset : blob.Offset+blob.Length] buf := pack[blob.Offset : blob.Offset+blob.Length]
@@ -366,16 +368,16 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
outputPrefix := "" outputPrefix := ""
filePrefix := "" filePrefix := ""
if err != nil { if err != nil {
printer.E("error decrypting blob: %v", err) Warnf("error decrypting blob: %v\n", err)
if opts.TryRepair || opts.RepairByte { if opts.TryRepair || opts.RepairByte {
plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte, printer) plaintext = tryRepairWithBitflip(key, buf, opts.RepairByte)
} }
if plaintext != nil { if plaintext != nil {
outputPrefix = "repaired " outputPrefix = "repaired "
filePrefix = "repaired-" filePrefix = "repaired-"
} else { } else {
plaintext = decryptUnsigned(key, buf) plaintext = decryptUnsigned(key, buf)
err = storePlainBlob(blob.ID, "damaged-", plaintext, printer) err = storePlainBlob(blob.ID, "damaged-", plaintext)
if err != nil { if err != nil {
return err return err
} }
@@ -386,7 +388,7 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
if blob.IsCompressed() { if blob.IsCompressed() {
decompressed, err := dec.DecodeAll(plaintext, nil) decompressed, err := dec.DecodeAll(plaintext, nil)
if err != nil { if err != nil {
printer.S(" failed to decompress blob %v", blob.ID) Printf(" failed to decompress blob %v\n", blob.ID)
} }
if decompressed != nil { if decompressed != nil {
plaintext = decompressed plaintext = decompressed
@@ -396,32 +398,37 @@ func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Reposi
id := restic.Hash(plaintext) id := restic.Hash(plaintext)
var prefix string var prefix string
if !id.Equal(blob.ID) { if !id.Equal(blob.ID) {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v", outputPrefix, len(plaintext), id, blob.ID) Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
prefix = "wrong-hash-" prefix = "wrong-hash-"
} else { } else {
printer.S(" successfully %vdecrypted blob (length %v), hash is %v, ID matches", outputPrefix, len(plaintext), id) Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
prefix = "correct-" prefix = "correct-"
} }
if opts.ExtractPack { if opts.ExtractPack {
err = storePlainBlob(id, filePrefix+prefix, plaintext, printer) err = storePlainBlob(id, filePrefix+prefix, plaintext)
if err != nil { if err != nil {
return err return err
} }
} }
if opts.ReuploadBlobs { if opts.ReuploadBlobs {
_, _, _, err := uploader.SaveBlob(ctx, blob.Type, plaintext, id, true) _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
if err != nil { if err != nil {
return err return err
} }
printer.S(" uploaded %v %v", blob.Type, id) Printf(" uploaded %v %v\n", blob.Type, id)
} }
} }
if opts.ReuploadBlobs {
return repo.Flush(ctx)
}
return nil return nil
}) })
return err
return wg.Wait()
} }
func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.Printer) error { func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
filename := fmt.Sprintf("%s%s.bin", prefix, id) filename := fmt.Sprintf("%s%s.bin", prefix, id)
f, err := os.Create(filename) f, err := os.Create(filename)
if err != nil { if err != nil {
@@ -439,18 +446,16 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte, printer progress.
return err return err
} }
printer.S("decrypt of blob %v stored at %v", id, filename) Printf("decrypt of blob %v stored at %v\n", id, filename)
return nil return nil
} }
func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamineOptions, args []string, term ui.Terminal) error { func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
printer := ui.NewProgressPrinter(false, gopts.Verbosity, term)
if opts.ExtractPack && gopts.NoLock { if opts.ExtractPack && gopts.NoLock {
return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive") return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive")
} }
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock, printer) ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock)
if err != nil { if err != nil {
return err return err
} }
@@ -462,7 +467,7 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
if err != nil { if err != nil {
id, err = restic.Find(ctx, repo, restic.PackFile, name) id, err = restic.Find(ctx, repo, restic.PackFile, name)
if err != nil { if err != nil {
printer.E("error: %v", err) Warnf("error: %v\n", err)
continue continue
} }
} }
@@ -473,15 +478,16 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
return errors.Fatal("no pack files to examine") return errors.Fatal("no pack files to examine")
} }
err = repo.LoadIndex(ctx, printer) bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil { if err != nil {
return err return err
} }
for _, id := range ids { for _, id := range ids {
err := examinePack(ctx, opts, repo, id, printer) err := examinePack(ctx, opts, repo, id)
if err != nil { if err != nil {
printer.E("error: %v", err) Warnf("error: %v\n", err)
} }
if err == context.Canceled { if err == context.Canceled {
break break
@@ -490,24 +496,24 @@ func runDebugExamine(ctx context.Context, gopts global.Options, opts DebugExamin
return nil return nil
} }
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID, printer progress.Printer) error { func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error {
printer.S("examine %v", id) Printf("examine %v\n", id)
buf, err := repo.LoadRaw(ctx, restic.PackFile, id) buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files // also process damaged pack files
if buf == nil { if buf == nil {
return err return err
} }
printer.S(" file size is %v", len(buf)) Printf(" file size is %v\n", len(buf))
gotID := restic.Hash(buf) gotID := restic.Hash(buf)
if !id.Equal(gotID) { if !id.Equal(gotID) {
printer.S(" wanted hash %v, got %v", id, gotID) Printf(" wanted hash %v, got %v\n", id, gotID)
} else { } else {
printer.S(" hash for file content matches") Printf(" hash for file content matches\n")
} }
printer.S(" ========================================") Printf(" ========================================\n")
printer.S(" looking for info in the indexes") Printf(" looking for info in the indexes\n")
blobsLoaded := false blobsLoaded := false
// examine all data the indexes have for the pack file // examine all data the indexes have for the pack file
@@ -517,41 +523,43 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo
continue continue
} }
checkPackSize(blobs, len(buf), printer) checkPackSize(blobs, len(buf))
err = loadBlobs(ctx, opts, repo, id, blobs, printer) err = loadBlobs(ctx, opts, repo, id, blobs)
if err != nil { if err != nil {
printer.E("error: %v", err) Warnf("error: %v\n", err)
} else { } else {
blobsLoaded = true blobsLoaded = true
} }
} }
printer.S(" ========================================") Printf(" ========================================\n")
printer.S(" inspect the pack itself") Printf(" inspect the pack itself\n")
blobs, _, err := repo.ListPack(ctx, id, int64(len(buf))) blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
if err != nil { if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err) return fmt.Errorf("pack %v: %v", id.Str(), err)
} }
checkPackSize(blobs, len(buf), printer) checkPackSize(blobs, len(buf))
if !blobsLoaded { if !blobsLoaded {
return loadBlobs(ctx, opts, repo, id, blobs, printer) return loadBlobs(ctx, opts, repo, id, blobs)
} }
return nil return nil
} }
func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) { func checkPackSize(blobs []restic.Blob, fileSize int) {
// track current size and offset // track current size and offset
var size, offset uint64 var size, offset uint64
blobs.Sort() sort.Slice(blobs, func(i, j int) bool {
return blobs[i].Offset < blobs[j].Offset
})
for _, pb := range blobs { for _, pb := range blobs {
printer.S(" %v blob %v, offset %-6d, raw length %-6d", pb.Type, pb.ID, pb.Offset, pb.Length) Printf(" %v blob %v, offset %-6d, raw length %-6d\n", pb.Type, pb.ID, pb.Offset, pb.Length)
if offset != uint64(pb.Offset) { if offset != uint64(pb.Offset) {
printer.S(" hole in file, want offset %v, got %v", offset, pb.Offset) Printf(" hole in file, want offset %v, got %v\n", offset, pb.Offset)
} }
offset = uint64(pb.Offset + pb.Length) offset = uint64(pb.Offset + pb.Length)
size += uint64(pb.Length) size += uint64(pb.Length)
@@ -559,8 +567,8 @@ func checkPackSize(blobs restic.Blobs, fileSize int, printer progress.Printer) {
size += uint64(pack.CalculateHeaderSize(blobs)) size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size { if uint64(fileSize) != size {
printer.S(" file sizes do not match: computed %v, file size is %v", size, fileSize) Printf(" file sizes do not match: computed %v, file size is %v\n", size, fileSize)
} else { } else {
printer.S(" file sizes match") Printf(" file sizes match\n")
} }
} }
+2 -5
View File
@@ -2,11 +2,8 @@
package main package main
import ( import "github.com/spf13/cobra"
"github.com/restic/restic/internal/global"
"github.com/spf13/cobra"
)
func registerDebugCommand(_ *cobra.Command, _ *global.Options) { func registerDebugCommand(_ *cobra.Command) {
// No commands to register in non-debug mode // No commands to register in non-debug mode
} }
+106 -102
View File
@@ -5,18 +5,17 @@ import (
"encoding/json" "encoding/json"
"path" "path"
"reflect" "reflect"
"sort"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
func newDiffCommand(globalOptions *global.Options) *cobra.Command { func newDiffCommand() *cobra.Command {
var opts DiffOptions var opts DiffOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
@@ -39,7 +38,7 @@ Metadata comparison will likely not work if a backup was created using the
To only compare files in specific subfolders, you can use the To only compare files in specific subfolders, you can use the
"snapshotID:subfolder" syntax, where "subfolder" is a path within the "snapshotID:subfolder" syntax, where "subfolder" is a path within the
snapshot tree as shown by "restic ls". snapshot.
EXIT STATUS EXIT STATUS
=========== ===========
@@ -53,7 +52,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
return runDiff(cmd.Context(), opts, *globalOptions, args, globalOptions.Term) return runDiff(cmd.Context(), opts, globalOptions, args)
}, },
} }
@@ -70,10 +69,10 @@ func (opts *DiffOptions) AddFlags(f *pflag.FlagSet) {
f.BoolVar(&opts.ShowMetadata, "metadata", false, "print changes in metadata") f.BoolVar(&opts.ShowMetadata, "metadata", false, "print changes in metadata")
} }
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*data.Snapshot, string, error) { func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
sn, subfolder, err := data.FindSnapshot(ctx, be, repo, desc) sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil { if err != nil {
return nil, "", errors.Fatalf("%s", err) return nil, "", errors.Fatal(err.Error())
} }
return sn, subfolder, err return sn, subfolder, err
} }
@@ -83,7 +82,6 @@ type Comparer struct {
repo restic.BlobLoader repo restic.BlobLoader
opts DiffOptions opts DiffOptions
printChange func(change *Change) printChange func(change *Change)
printError func(string, ...interface{})
} }
type Change struct { type Change struct {
@@ -107,15 +105,15 @@ type DiffStat struct {
} }
// Add adds stats information for node to s. // Add adds stats information for node to s.
func (s *DiffStat) Add(node *data.Node) { func (s *DiffStat) Add(node *restic.Node) {
if node == nil { if node == nil {
return return
} }
switch node.Type { switch node.Type {
case data.NodeTypeFile: case restic.NodeTypeFile:
s.Files++ s.Files++
case data.NodeTypeDir: case restic.NodeTypeDir:
s.Dirs++ s.Dirs++
default: default:
s.Others++ s.Others++
@@ -123,13 +121,13 @@ func (s *DiffStat) Add(node *data.Node) {
} }
// addBlobs adds the blobs of node to s. // addBlobs adds the blobs of node to s.
func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) { func addBlobs(bs restic.BlobSet, node *restic.Node) {
if node == nil { if node == nil {
return return
} }
switch node.Type { switch node.Type {
case data.NodeTypeFile: case restic.NodeTypeFile:
for _, blob := range node.Content { for _, blob := range node.Content {
h := restic.BlobHandle{ h := restic.BlobHandle{
ID: blob, ID: blob,
@@ -137,7 +135,7 @@ func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
} }
bs.Insert(h) bs.Insert(h)
} }
case data.NodeTypeDir: case restic.NodeTypeDir:
h := restic.BlobHandle{ h := restic.BlobHandle{
ID: *node.Subtree, ID: *node.Subtree,
Type: restic.TreeBlob, Type: restic.TreeBlob,
@@ -147,18 +145,18 @@ func addBlobs(bs restic.AssociatedBlobSet, node *data.Node) {
} }
type DiffStatsContainer struct { type DiffStatsContainer struct {
MessageType string `json:"message_type"` // "statistics" MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"` SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"` TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"` ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"` Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"` Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.AssociatedBlobSet `json:"-"` BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
} }
// updateBlobs updates the blob counters in the stats struct. // updateBlobs updates the blob counters in the stats struct.
func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *DiffStat, printError func(string, ...interface{})) { func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
for h := range blobs.Keys() { for h := range blobs {
switch h.Type { switch h.Type {
case restic.DataBlob: case restic.DataBlob:
stats.DataBlobs++ stats.DataBlobs++
@@ -168,7 +166,7 @@ func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *Diff
size, found := repo.LookupBlobSize(h.Type, h.ID) size, found := repo.LookupBlobSize(h.Type, h.ID)
if !found { if !found {
printError("unable to find blob size for %v", h) Warnf("unable to find blob size for %v\n", h)
continue continue
} }
@@ -176,33 +174,30 @@ func updateBlobs(repo restic.Loader, blobs restic.AssociatedBlobSet, stats *Diff
} }
} }
func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.AssociatedBlobSet, prefix string, id restic.ID) error { func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
debug.Log("print %v tree %v", mode, id) debug.Log("print %v tree %v", mode, id)
tree, err := data.LoadTree(ctx, c.repo, id) tree, err := restic.LoadTree(ctx, c.repo, id)
if err != nil { if err != nil {
return err return err
} }
for item := range tree { for _, node := range tree.Nodes {
if item.Error != nil {
return item.Error
}
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
} }
node := item.Node
name := path.Join(prefix, node.Name) name := path.Join(prefix, node.Name)
if node.Type == data.NodeTypeDir { if node.Type == restic.NodeTypeDir {
name += "/" name += "/"
} }
c.printChange(NewChange(name, mode)) c.printChange(NewChange(name, mode))
stats.Add(node) stats.Add(node)
addBlobs(blobs, node) addBlobs(blobs, node)
if node.Type == data.NodeTypeDir { if node.Type == restic.NodeTypeDir {
err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree) err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
if err != nil && err != context.Canceled { if err != nil && err != context.Canceled {
c.printError("error: %v", err) Warnf("error: %v\n", err)
} }
} }
} }
@@ -210,28 +205,24 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
return ctx.Err() return ctx.Err()
} }
func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSet, id restic.ID) error { func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
debug.Log("print tree %v", id) debug.Log("print tree %v", id)
tree, err := data.LoadTree(ctx, c.repo, id) tree, err := restic.LoadTree(ctx, c.repo, id)
if err != nil { if err != nil {
return err return err
} }
for item := range tree { for _, node := range tree.Nodes {
if item.Error != nil {
return item.Error
}
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
} }
node := item.Node
addBlobs(blobs, node) addBlobs(blobs, node)
if node.Type == data.NodeTypeDir { if node.Type == restic.NodeTypeDir {
err := c.collectDir(ctx, blobs, *node.Subtree) err := c.collectDir(ctx, blobs, *node.Subtree)
if err != nil && err != context.Canceled { if err != nil && err != context.Canceled {
c.printError("error: %v", err) Warnf("error: %v\n", err)
} }
} }
} }
@@ -239,41 +230,56 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.AssociatedBlobSe
return ctx.Err() return ctx.Err()
} }
func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) {
names := make(map[string]struct{})
tree1Nodes = make(map[string]*restic.Node)
for _, node := range tree1.Nodes {
tree1Nodes[node.Name] = node
names[node.Name] = struct{}{}
}
tree2Nodes = make(map[string]*restic.Node)
for _, node := range tree2.Nodes {
tree2Nodes[node.Name] = node
names[node.Name] = struct{}{}
}
uniqueNames = make([]string, 0, len(names))
for name := range names {
uniqueNames = append(uniqueNames, name)
}
sort.Strings(uniqueNames)
return tree1Nodes, tree2Nodes, uniqueNames
}
func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error { func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
debug.Log("diffing %v to %v", id1, id2) debug.Log("diffing %v to %v", id1, id2)
tree1, err := data.LoadTree(ctx, c.repo, id1) tree1, err := restic.LoadTree(ctx, c.repo, id1)
if err != nil { if err != nil {
return err return err
} }
tree2, err := data.LoadTree(ctx, c.repo, id2) tree2, err := restic.LoadTree(ctx, c.repo, id2)
if err != nil { if err != nil {
return err return err
} }
for dt := range data.DualTreeIterator(tree1, tree2) { tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2)
if dt.Error != nil {
return dt.Error for _, name := range names {
}
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
} }
node1 := dt.Tree1 node1, t1 := tree1Nodes[name]
node2 := dt.Tree2 node2, t2 := tree2Nodes[name]
var name string
if node1 != nil {
name = node1.Name
} else {
name = node2.Name
}
addBlobs(stats.BlobsBefore, node1) addBlobs(stats.BlobsBefore, node1)
addBlobs(stats.BlobsAfter, node2) addBlobs(stats.BlobsAfter, node2)
switch { switch {
case node1 != nil && node2 != nil: case t1 && t2:
name := path.Join(prefix, name) name := path.Join(prefix, name)
mod := "" mod := ""
@@ -281,12 +287,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
mod += "T" mod += "T"
} }
if node2.Type == data.NodeTypeDir { if node2.Type == restic.NodeTypeDir {
name += "/" name += "/"
} }
if node1.Type == data.NodeTypeFile && if node1.Type == restic.NodeTypeFile &&
node2.Type == data.NodeTypeFile && node2.Type == restic.NodeTypeFile &&
!reflect.DeepEqual(node1.Content, node2.Content) { !reflect.DeepEqual(node1.Content, node2.Content) {
mod += "M" mod += "M"
stats.ChangedFiles++ stats.ChangedFiles++
@@ -308,7 +314,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
c.printChange(NewChange(name, mod)) c.printChange(NewChange(name, mod))
} }
if node1.Type == data.NodeTypeDir && node2.Type == data.NodeTypeDir { if node1.Type == restic.NodeTypeDir && node2.Type == restic.NodeTypeDir {
var err error var err error
if (*node1.Subtree).Equal(*node2.Subtree) { if (*node1.Subtree).Equal(*node2.Subtree) {
err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree) err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree)
@@ -316,35 +322,35 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
err = c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree) err = c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree)
} }
if err != nil && err != context.Canceled { if err != nil && err != context.Canceled {
c.printError("error: %v", err) Warnf("error: %v\n", err)
} }
} }
case node1 != nil && node2 == nil: case t1 && !t2:
prefix := path.Join(prefix, name) prefix := path.Join(prefix, name)
if node1.Type == data.NodeTypeDir { if node1.Type == restic.NodeTypeDir {
prefix += "/" prefix += "/"
} }
c.printChange(NewChange(prefix, "-")) c.printChange(NewChange(prefix, "-"))
stats.Removed.Add(node1) stats.Removed.Add(node1)
if node1.Type == data.NodeTypeDir { if node1.Type == restic.NodeTypeDir {
err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree) err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
if err != nil && err != context.Canceled { if err != nil && err != context.Canceled {
c.printError("error: %v", err) Warnf("error: %v\n", err)
} }
} }
case node1 == nil && node2 != nil: case !t1 && t2:
prefix := path.Join(prefix, name) prefix := path.Join(prefix, name)
if node2.Type == data.NodeTypeDir { if node2.Type == restic.NodeTypeDir {
prefix += "/" prefix += "/"
} }
c.printChange(NewChange(prefix, "+")) c.printChange(NewChange(prefix, "+"))
stats.Added.Add(node2) stats.Added.Add(node2)
if node2.Type == data.NodeTypeDir { if node2.Type == restic.NodeTypeDir {
err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree) err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
if err != nil && err != context.Canceled { if err != nil && err != context.Canceled {
c.printError("error: %v", err) Warnf("error: %v\n", err)
} }
} }
} }
@@ -353,14 +359,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
return ctx.Err() return ctx.Err()
} }
func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args []string, term ui.Terminal) error { func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 { if len(args) != 2 {
return errors.Fatalf("specify two snapshot IDs") return errors.Fatalf("specify two snapshot IDs")
} }
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer)
if err != nil { if err != nil {
return err return err
} }
@@ -382,9 +386,10 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
} }
if !gopts.JSON { if !gopts.JSON {
printer.P("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
} }
if err = repo.LoadIndex(ctx, printer); err != nil { bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err = repo.LoadIndex(ctx, bar); err != nil {
return err return err
} }
@@ -396,31 +401,30 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str()) return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
} }
sn1.Tree, err = data.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1) sn1.Tree, err = restic.FindTreeDirectory(ctx, repo, sn1.Tree, subfolder1)
if err != nil { if err != nil {
return err return err
} }
sn2.Tree, err = data.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2) sn2.Tree, err = restic.FindTreeDirectory(ctx, repo, sn2.Tree, subfolder2)
if err != nil { if err != nil {
return err return err
} }
c := &Comparer{ c := &Comparer{
repo: repo, repo: repo,
opts: opts, opts: opts,
printError: printer.E,
printChange: func(change *Change) { printChange: func(change *Change) {
printer.S("%-5s%v", change.Modifier, change.Path) Printf("%-5s%v\n", change.Modifier, change.Path)
}, },
} }
if gopts.JSON { if gopts.JSON {
enc := json.NewEncoder(gopts.Term.OutputWriter()) enc := json.NewEncoder(globalOptions.stdout)
c.printChange = func(change *Change) { c.printChange = func(change *Change) {
err := enc.Encode(change) err := enc.Encode(change)
if err != nil { if err != nil {
printer.E("JSON encode failed: %v", err) Warnf("JSON encode failed: %v\n", err)
} }
} }
} }
@@ -433,9 +437,9 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
MessageType: "statistics", MessageType: "statistics",
SourceSnapshot: args[0], SourceSnapshot: args[0],
TargetSnapshot: args[1], TargetSnapshot: args[1],
BlobsBefore: repo.NewAssociatedBlobSet(), BlobsBefore: restic.NewBlobSet(),
BlobsAfter: repo.NewAssociatedBlobSet(), BlobsAfter: restic.NewBlobSet(),
BlobsCommon: repo.NewAssociatedBlobSet(), BlobsCommon: restic.NewBlobSet(),
} }
stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree}) stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree}) stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
@@ -446,23 +450,23 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts global.Options, args [
} }
both := stats.BlobsBefore.Intersect(stats.BlobsAfter) both := stats.BlobsBefore.Intersect(stats.BlobsAfter)
updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed, printer.E) updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added, printer.E) updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
if gopts.JSON { if gopts.JSON {
err := json.NewEncoder(gopts.Term.OutputWriter()).Encode(stats) err := json.NewEncoder(globalOptions.stdout).Encode(stats)
if err != nil { if err != nil {
printer.E("JSON encode failed: %v", err) Warnf("JSON encode failed: %v\n", err)
} }
} else { } else {
printer.S("") Printf("\n")
printer.S("Files: %5d new, %5d removed, %5d changed", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles) Printf("Files: %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
printer.S("Dirs: %5d new, %5d removed", stats.Added.Dirs, stats.Removed.Dirs) Printf("Dirs: %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
printer.S("Others: %5d new, %5d removed", stats.Added.Others, stats.Removed.Others) Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
printer.S("Data Blobs: %5d new, %5d removed", stats.Added.DataBlobs, stats.Removed.DataBlobs) Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
printer.S("Tree Blobs: %5d new, %5d removed", stats.Added.TreeBlobs, stats.Removed.TreeBlobs) Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
printer.S(" Added: %-5s", ui.FormatBytes(stats.Added.Bytes)) Printf(" Added: %-5s\n", ui.FormatBytes(stats.Added.Bytes))
printer.S(" Removed: %-5s", ui.FormatBytes(stats.Removed.Bytes)) Printf(" Removed: %-5s\n", ui.FormatBytes(stats.Removed.Bytes))
} }
return nil return nil
+8 -9
View File
@@ -11,16 +11,15 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func testRunDiffOutput(t testing.TB, gopts global.Options, firstSnapshotID string, secondSnapshotID string) (string, error) { func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error { buf, err := withCaptureStdout(func() error {
opts := DiffOptions{ opts := DiffOptions{
ShowMetadata: false, ShowMetadata: false,
} }
return runDiff(ctx, opts, gopts, []string{firstSnapshotID, secondSnapshotID}, gopts.Term) return runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
}) })
return buf.String(), err return buf.String(), err
} }
@@ -124,10 +123,10 @@ func TestDiff(t *testing.T) {
// quiet suppresses the diff output except for the summary // quiet suppresses the diff output except for the summary
env.gopts.Quiet = false env.gopts.Quiet = false
_, err := testRunDiffOutput(t, env.gopts, "", secondSnapshotID) _, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
rtest.Assert(t, err != nil, "expected error on invalid snapshot id") rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
out, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID) out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err) rtest.OK(t, err)
for _, pattern := range diffOutputRegexPatterns { for _, pattern := range diffOutputRegexPatterns {
@@ -138,7 +137,7 @@ func TestDiff(t *testing.T) {
// check quiet output // check quiet output
env.gopts.Quiet = true env.gopts.Quiet = true
outQuiet, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID) outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err) rtest.OK(t, err)
rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out)) rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
@@ -155,7 +154,7 @@ func TestDiffJSON(t *testing.T) {
// quiet suppresses the diff output except for the summary // quiet suppresses the diff output except for the summary
env.gopts.Quiet = false env.gopts.Quiet = false
env.gopts.JSON = true env.gopts.JSON = true
out, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID) out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err) rtest.OK(t, err)
var stat DiffStatsContainer var stat DiffStatsContainer
@@ -182,7 +181,7 @@ func TestDiffJSON(t *testing.T) {
// check quiet output // check quiet output
env.gopts.Quiet = true env.gopts.Quiet = true
outQuiet, err := testRunDiffOutput(t, env.gopts, firstSnapshotID, secondSnapshotID) outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err) rtest.OK(t, err)
stat = DiffStatsContainer{} stat = DiffStatsContainer{}
+31 -35
View File
@@ -7,23 +7,20 @@ import (
"path" "path"
"path/filepath" "path/filepath"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/dump" "github.com/restic/restic/internal/dump"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
func newDumpCommand(globalOptions *global.Options) *cobra.Command { func newDumpCommand() *cobra.Command {
var opts DumpOptions var opts DumpOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "dump [flags] snapshotID file", Use: "dump [flags] snapshotID file",
Short: "Print backed-up files or folders to stdout", Short: "Print a backed-up file to stdout",
Long: ` Long: `
The "dump" command extracts files from a snapshot from the repository. If a The "dump" command extracts files from a snapshot from the repository. If a
single file is selected, it prints its contents to stdout. Folders are output single file is selected, it prints its contents to stdout. Folders are output
@@ -35,7 +32,7 @@ repository.
To include the folder content at the root of the archive, you can use the To include the folder content at the root of the archive, you can use the
"snapshotID:subfolder" syntax, where "subfolder" is a path within the "snapshotID:subfolder" syntax, where "subfolder" is a path within the
snapshot tree as shown by "restic ls". snapshot.
EXIT STATUS EXIT STATUS
=========== ===========
@@ -49,8 +46,7 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter) return runDump(cmd.Context(), opts, globalOptions, args)
return runDump(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
}, },
} }
@@ -60,7 +56,7 @@ Exit status is 12 if the password is incorrect.
// DumpOptions collects all options for the dump command. // DumpOptions collects all options for the dump command.
type DumpOptions struct { type DumpOptions struct {
data.SnapshotFilter restic.SnapshotFilter
Archive string Archive string
Target string Target string
} }
@@ -80,7 +76,7 @@ func splitPath(p string) []string {
return append(s, f) return append(s, f)
} }
func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
// If we print / we need to assume that there are multiple nodes at that // If we print / we need to assume that there are multiple nodes at that
// level in the tree. // level in the tree.
if pathComponents[0] == "" { if pathComponents[0] == "" {
@@ -92,38 +88,35 @@ func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.
item := filepath.Join(prefix, pathComponents[0]) item := filepath.Join(prefix, pathComponents[0])
l := len(pathComponents) l := len(pathComponents)
for it := range tree { for _, node := range tree.Nodes {
if it.Error != nil {
return it.Error
}
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
} }
node := it.Node
// If dumping something in the highest level it will just take the // If dumping something in the highest level it will just take the
// first item it finds and dump that according to the switch case below. // first item it finds and dump that according to the switch case below.
if node.Name == pathComponents[0] { if node.Name == pathComponents[0] {
switch { switch {
case l == 1 && node.Type == data.NodeTypeFile: case l == 1 && node.Type == restic.NodeTypeFile:
return d.WriteNode(ctx, node) return d.WriteNode(ctx, node)
case l > 1 && node.Type == data.NodeTypeDir: case l > 1 && node.Type == restic.NodeTypeDir:
subtree, err := data.LoadTree(ctx, repo, *node.Subtree) subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil { if err != nil {
return errors.Wrapf(err, "cannot load subtree for %q", item) return errors.Wrapf(err, "cannot load subtree for %q", item)
} }
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc) return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
case node.Type == data.NodeTypeDir: case node.Type == restic.NodeTypeDir:
if err := canWriteArchiveFunc(); err != nil { if err := canWriteArchiveFunc(); err != nil {
return err return err
} }
subtree, err := data.LoadTree(ctx, repo, *node.Subtree) subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil { if err != nil {
return err return err
} }
return d.DumpTree(ctx, subtree, item) return d.DumpTree(ctx, subtree, item)
case l > 1: case l > 1:
return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type) return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type)
case node.Type != data.NodeTypeFile: case node.Type != restic.NodeTypeFile:
return fmt.Errorf("%q should be a file, but is a %q", item, node.Type) return fmt.Errorf("%q should be a file, but is a %q", item, node.Type)
} }
} }
@@ -131,13 +124,11 @@ func printFromTree(ctx context.Context, tree data.TreeNodeIterator, repo restic.
return fmt.Errorf("path %q not found in snapshot", item) return fmt.Errorf("path %q not found in snapshot", item)
} }
func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args []string, term ui.Terminal) error { func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 { if len(args) != 2 {
return errors.Fatal("no file and no snapshot ID specified") return errors.Fatal("no file and no snapshot ID specified")
} }
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
switch opts.Archive { switch opts.Archive {
case "tar", "zip": case "tar", "zip":
default: default:
@@ -151,34 +142,39 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [
splittedPath := splitPath(path.Clean(pathToPrint)) splittedPath := splitPath(path.Clean(pathToPrint))
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil { if err != nil {
return err return err
} }
defer unlock() defer unlock()
sn, subfolder, err := opts.SnapshotFilter.FindLatest(ctx, repo, repo, snapshotIDString) sn, subfolder, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo, repo, snapshotIDString)
if err != nil { if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err) return errors.Fatalf("failed to find snapshot: %v", err)
} }
err = repo.LoadIndex(ctx, printer) bar := newIndexProgress(gopts.Quiet, gopts.JSON)
err = repo.LoadIndex(ctx, bar)
if err != nil { if err != nil {
return err return err
} }
sn.Tree, err = data.FindTreeDirectory(ctx, repo, sn.Tree, subfolder) sn.Tree, err = restic.FindTreeDirectory(ctx, repo, sn.Tree, subfolder)
if err != nil { if err != nil {
return err return err
} }
tree, err := data.LoadTree(ctx, repo, *sn.Tree) tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
if err != nil { if err != nil {
return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err) return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
} }
outputFileWriter := term.OutputRaw() outputFileWriter := os.Stdout
canWriteArchiveFunc := checkStdoutArchive(term) canWriteArchiveFunc := checkStdoutArchive
if opts.Target != "" { if opts.Target != "" {
file, err := os.Create(opts.Target) file, err := os.Create(opts.Target)
@@ -202,9 +198,9 @@ func runDump(ctx context.Context, opts DumpOptions, gopts global.Options, args [
return nil return nil
} }
func checkStdoutArchive(term ui.Terminal) func() error { func checkStdoutArchive() error {
if term.OutputIsTerminal() { if stdoutIsTerminal() {
return func() error { return fmt.Errorf("stdout is the terminal, please redirect output") } return fmt.Errorf("stdout is the terminal, please redirect output")
} }
return func() error { return nil } return nil
} }
+5 -4
View File
@@ -1,15 +1,16 @@
package main package main
import ( import (
"fmt"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/feature"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/ui/table" "github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
func newFeaturesCommand(globalOptions *global.Options) *cobra.Command { func newFeaturesCommand() *cobra.Command {
cmd := &cobra.Command{ cmd := &cobra.Command{
Use: "features", Use: "features",
Short: "Print list of feature flags", Short: "Print list of feature flags",
@@ -38,7 +39,7 @@ Exit status is 1 if there was any error.
return errors.Fatal("the feature command expects no arguments") return errors.Fatal("the feature command expects no arguments")
} }
globalOptions.Term.Print("All Feature Flags:\n") fmt.Printf("All Feature Flags:\n")
flags := feature.Flag.List() flags := feature.Flag.List()
tab := table.New() tab := table.New()
@@ -50,7 +51,7 @@ Exit status is 1 if there was any error.
for _, flag := range flags { for _, flag := range flags {
tab.AddRow(flag) tab.AddRow(flag)
} }
return tab.Write(globalOptions.Term.OutputWriter()) return tab.Write(globalOptions.stdout)
}, },
} }
+66 -106
View File
@@ -3,8 +3,6 @@ package main
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"io"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -12,20 +10,14 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/walker" "github.com/restic/restic/internal/walker"
) )
// errFindDone is returned from the tree walk when all requested tree IDs were found. func newFindCommand() *cobra.Command {
var errFindDone = errors.New("find: all tree IDs found")
func newFindCommand(globalOptions *global.Options) *cobra.Command {
var opts FindOptions var opts FindOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
@@ -33,11 +25,16 @@ func newFindCommand(globalOptions *global.Options) *cobra.Command {
Short: "Find a file, a directory or restic IDs", Short: "Find a file, a directory or restic IDs",
Long: ` Long: `
The "find" command searches for files or directories in snapshots stored in the The "find" command searches for files or directories in snapshots stored in the
repository. It can also be used to search for restic blobs, trees or pack repo.
files for troubleshooting. It can also be used to search for restic blobs or trees for troubleshooting.
The default sort option for the snapshots is youngest to oldest. To sort the The default sort option for the snapshots is youngest to oldest. To sort the
output from oldest to youngest specify --reverse. output from oldest to youngest specify --reverse.`,
Example: `restic find config.json
restic find --json "*.yml" "*.json"
restic find --json --blob 420f620f b46ebe8a ddd38656
restic find --show-pack-id --blob 420f620f
restic find --tree 577c2bc9 f81f2e22 a62827a9
restic find --pack 025c1d06
EXIT STATUS EXIT STATUS
=========== ===========
@@ -48,17 +45,10 @@ Exit status is 10 if the repository does not exist.
Exit status is 11 if the repository is already locked. Exit status is 11 if the repository is already locked.
Exit status is 12 if the password is incorrect. Exit status is 12 if the password is incorrect.
`, `,
Example: `restic find config.json
restic find --json "*.yml" "*.json"
restic find --json --blob 420f620f b46ebe8a ddd38656
restic find --show-pack-id --blob 420f620f
restic find --tree 577c2bc9 f81f2e22 a62827a9
restic find --pack 025c1d06`,
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter) return runFind(cmd.Context(), opts, globalOptions, args)
return runFind(cmd.Context(), opts, *globalOptions, args, globalOptions.Term)
}, },
} }
@@ -77,7 +67,7 @@ type FindOptions struct {
ListLong bool ListLong bool
HumanReadable bool HumanReadable bool
Reverse bool Reverse bool
data.SnapshotFilter restic.SnapshotFilter
} }
func (opts *FindOptions) AddFlags(f *pflag.FlagSet) { func (opts *FindOptions) AddFlags(f *pflag.FlagSet) {
@@ -131,19 +121,13 @@ type statefulOutput struct {
HumanReadable bool HumanReadable bool
JSON bool JSON bool
inuse bool inuse bool
newsn *data.Snapshot newsn *restic.Snapshot
oldsn *data.Snapshot oldsn *restic.Snapshot
hits int hits int
printer interface {
S(string, ...interface{})
P(string, ...interface{})
E(string, ...interface{})
}
stdout io.Writer
} }
func (s *statefulOutput) PrintPatternJSON(path string, node *data.Node) { func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
type findNode data.Node type findNode restic.Node
b, err := json.Marshal(struct { b, err := json.Marshal(struct {
// Add these attributes // Add these attributes
Path string `json:"path,omitempty"` Path string `json:"path,omitempty"`
@@ -164,40 +148,40 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *data.Node) {
findNode: (*findNode)(node), findNode: (*findNode)(node),
}) })
if err != nil { if err != nil {
s.printer.E("Marshal failed: %v", err) Warnf("Marshall failed: %v\n", err)
return return
} }
if !s.inuse { if !s.inuse {
_, _ = s.stdout.Write([]byte("[")) Printf("[")
s.inuse = true s.inuse = true
} }
if s.newsn != s.oldsn { if s.newsn != s.oldsn {
if s.oldsn != nil { if s.oldsn != nil {
_, _ = fmt.Fprintf(s.stdout, "],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID()) Printf("],\"hits\":%d,\"snapshot\":%q},", s.hits, s.oldsn.ID())
} }
_, _ = s.stdout.Write([]byte(`{"matches":[`)) Printf(`{"matches":[`)
s.oldsn = s.newsn s.oldsn = s.newsn
s.hits = 0 s.hits = 0
} }
if s.hits > 0 { if s.hits > 0 {
_, _ = s.stdout.Write([]byte(",")) Printf(",")
} }
_, _ = s.stdout.Write(b) Print(string(b))
s.hits++ s.hits++
} }
func (s *statefulOutput) PrintPatternNormal(path string, node *data.Node) { func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) {
if s.newsn != s.oldsn { if s.newsn != s.oldsn {
if s.oldsn != nil { if s.oldsn != nil {
s.printer.P("") Verbosef("\n")
} }
s.oldsn = s.newsn s.oldsn = s.newsn
s.printer.P("Found matching entries in snapshot %s from %s", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(global.TimeFormat)) Verbosef("Found matching entries in snapshot %s from %s\n", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(TimeFormat))
} }
s.printer.S(formatNode(path, node, s.ListLong, s.HumanReadable)) Println(formatNode(path, node, s.ListLong, s.HumanReadable))
} }
func (s *statefulOutput) PrintPattern(path string, node *data.Node) { func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
if s.JSON { if s.JSON {
s.PrintPatternJSON(path, node) s.PrintPatternJSON(path, node)
} else { } else {
@@ -205,7 +189,7 @@ func (s *statefulOutput) PrintPattern(path string, node *data.Node) {
} }
} }
func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *data.Snapshot) { func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
b, err := json.Marshal(struct { b, err := json.Marshal(struct {
// Add these attributes // Add these attributes
ObjectType string `json:"object_type"` ObjectType string `json:"object_type"`
@@ -223,32 +207,32 @@ func (s *statefulOutput) PrintObjectJSON(kind, id, nodepath, treeID string, sn *
Time: sn.Time, Time: sn.Time,
}) })
if err != nil { if err != nil {
s.printer.E("Marshal failed: %v", err) Warnf("Marshall failed: %v\n", err)
return return
} }
if !s.inuse { if !s.inuse {
_, _ = s.stdout.Write([]byte("[")) Printf("[")
s.inuse = true s.inuse = true
} }
if s.hits > 0 { if s.hits > 0 {
_, _ = s.stdout.Write([]byte(",")) Printf(",")
} }
_, _ = s.stdout.Write(b) Print(string(b))
s.hits++ s.hits++
} }
func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *data.Snapshot) { func (s *statefulOutput) PrintObjectNormal(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
s.printer.S("Found %s %s", kind, id) Printf("Found %s %s\n", kind, id)
if kind == "blob" { if kind == "blob" {
s.printer.S(" ... in file %s", nodepath) Printf(" ... in file %s\n", nodepath)
s.printer.S(" (tree %s)", treeID) Printf(" (tree %s)\n", treeID)
} else { } else {
s.printer.S(" ... path %s", nodepath) Printf(" ... path %s\n", nodepath)
} }
s.printer.S(" ... in snapshot %s (%s)", sn.ID().Str(), sn.Time.Local().Format(global.TimeFormat)) Printf(" ... in snapshot %s (%s)\n", sn.ID().Str(), sn.Time.Local().Format(TimeFormat))
} }
func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *data.Snapshot) { func (s *statefulOutput) PrintObject(kind, id, nodepath, treeID string, sn *restic.Snapshot) {
if s.JSON { if s.JSON {
s.PrintObjectJSON(kind, id, nodepath, treeID, sn) s.PrintObjectJSON(kind, id, nodepath, treeID, sn)
} else { } else {
@@ -260,12 +244,12 @@ func (s *statefulOutput) Finish() {
if s.JSON { if s.JSON {
// do some finishing up // do some finishing up
if s.oldsn != nil { if s.oldsn != nil {
_, _ = fmt.Fprintf(s.stdout, "],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID()) Printf("],\"hits\":%d,\"snapshot\":%q}", s.hits, s.oldsn.ID())
} }
if s.inuse { if s.inuse {
_, _ = s.stdout.Write([]byte("]\n")) Printf("]\n")
} else { } else {
_, _ = s.stdout.Write([]byte("[]\n")) Printf("[]\n")
} }
return return
} }
@@ -279,14 +263,9 @@ type Finder struct {
blobIDs map[string]struct{} blobIDs map[string]struct{}
treeIDs map[string]struct{} treeIDs map[string]struct{}
itemsFound int itemsFound int
printer interface {
S(string, ...interface{})
P(string, ...interface{})
E(string, ...interface{})
}
} }
func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error { func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest) debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
if sn.Tree == nil { if sn.Tree == nil {
@@ -294,12 +273,11 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error {
} }
f.out.newsn = sn f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error { return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil { if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err) debug.Log("Error loading tree %v: %v", parentTreeID, err)
f.printer.S("Unable to load tree %s", parentTreeID) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
f.printer.S(" ... which belongs to snapshot %s", sn.ID())
return walker.ErrSkipNode return walker.ErrSkipNode
} }
@@ -327,7 +305,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *data.Snapshot) error {
} }
var errIfNoMatch error var errIfNoMatch error
if node.Type == data.NodeTypeDir { if node.Type == restic.NodeTypeDir {
var childMayMatch bool var childMayMatch bool
for _, pat := range f.pat.pattern { for _, pat := range f.pat.pattern {
mayMatch, err := filter.ChildMatch(pat, normalizedNodepath) mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
@@ -379,13 +357,13 @@ func (f *Finder) findTree(treeID restic.ID, nodepath string) error {
// looking for blobs) // looking for blobs)
if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil { if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
// Return an error to terminate the Walk // Return an error to terminate the Walk
return errFindDone return errors.New("OK")
} }
} }
return nil return nil
} }
func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error { func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
debug.Log("searching IDs in snapshot %s", sn.ID()) debug.Log("searching IDs in snapshot %s", sn.ID())
if sn.Tree == nil { if sn.Tree == nil {
@@ -393,12 +371,11 @@ func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error {
} }
f.out.newsn = sn f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *data.Node, err error) error { return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil { if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err) debug.Log("Error loading tree %v: %v", parentTreeID, err)
f.printer.S("Unable to load tree %s", parentTreeID) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
f.printer.S(" ... which belongs to snapshot %s", sn.ID())
return walker.ErrSkipNode return walker.ErrSkipNode
} }
@@ -418,7 +395,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *data.Snapshot) error {
} }
} }
if node.Type == data.NodeTypeFile && f.blobIDs != nil { if node.Type == restic.NodeTypeFile && f.blobIDs != nil {
for _, id := range node.Content { for _, id := range node.Content {
if ctx.Err() != nil { if ctx.Err() != nil {
return ctx.Err() return ctx.Err()
@@ -454,9 +431,6 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
if f.blobIDs == nil { if f.blobIDs == nil {
f.blobIDs = make(map[string]struct{}) f.blobIDs = make(map[string]struct{})
} }
if f.treeIDs == nil {
f.treeIDs = make(map[string]struct{})
}
debug.Log("Looking for packs...") debug.Log("Looking for packs...")
err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { err := f.repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
@@ -477,14 +451,7 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
return err return err
} }
for _, b := range blobs { for _, b := range blobs {
switch b.Type { f.blobIDs[b.ID.String()] = struct{}{}
case restic.DataBlob:
f.blobIDs[b.ID.String()] = struct{}{}
case restic.TreeBlob:
f.treeIDs[b.ID.String()] = struct{}{}
default:
panic(fmt.Sprintf("unknown type %v in blob list", b.Type.String()))
}
} }
// Stop searching when all packs have been found // Stop searching when all packs have been found
if len(packIDs) == 0 { if len(packIDs) == 0 {
@@ -557,7 +524,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
for h := range indexPackIDs { for h := range indexPackIDs {
list = append(list, h) list = append(list, h)
} }
f.printer.E("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list) Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
} }
return packIDs, nil return packIDs, nil
} }
@@ -565,20 +532,19 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
func (f *Finder) findObjectPack(id string, t restic.BlobType) { func (f *Finder) findObjectPack(id string, t restic.BlobType) {
rid, err := restic.ParseID(id) rid, err := restic.ParseID(id)
if err != nil { if err != nil {
f.printer.S("Note: cannot find pack for object '%s', unable to parse ID: %v", id, err) Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err)
return return
} }
blobs := f.repo.LookupBlob(t, rid) blobs := f.repo.LookupBlob(t, rid)
if len(blobs) == 0 { if len(blobs) == 0 {
f.printer.S("Object %s with type %s not found in the index", t.String(), rid.Str()) Printf("Object %s not found in the index\n", rid.Str())
return return
} }
for _, b := range blobs { for _, b := range blobs {
if b.ID.Equal(rid) { if b.ID.Equal(rid) {
f.printer.S("Object belongs to pack %s", b.PackID) Printf("Object belongs to pack %s\n ... Pack %s: %s\n", b.PackID, b.PackID.Str(), b.String())
f.printer.S(" ... Pack %s: %s", b.PackID.Str(), b.String())
break break
} }
} }
@@ -594,13 +560,11 @@ func (f *Finder) findObjectsPacks() {
} }
} }
func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args []string, term ui.Terminal) error { func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 { if len(args) == 0 {
return errors.Fatal("wrong number of arguments") return errors.Fatal("wrong number of arguments")
} }
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term)
var err error var err error
pat := findPattern{pattern: args} pat := findPattern{pattern: args}
if opts.CaseInsensitive { if opts.CaseInsensitive {
@@ -622,10 +586,6 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
} }
} }
if !pat.newest.IsZero() && !pat.oldest.IsZero() && pat.oldest.After(pat.newest) {
return errors.Fatal("--oldest must specify a time before --newest")
}
// Check at most only one kind of IDs is provided: currently we // Check at most only one kind of IDs is provided: currently we
// can't mix types // can't mix types
if (opts.BlobID && opts.TreeID) || if (opts.BlobID && opts.TreeID) ||
@@ -634,7 +594,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
return errors.Fatal("cannot have several ID types") return errors.Fatal("cannot have several ID types")
} }
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock, printer) ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
if err != nil { if err != nil {
return err return err
} }
@@ -644,15 +604,15 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
if err != nil { if err != nil {
return err return err
} }
if err = repo.LoadIndex(ctx, printer); err != nil { bar := newIndexProgress(gopts.Quiet, gopts.JSON)
if err = repo.LoadIndex(ctx, bar); err != nil {
return err return err
} }
f := &Finder{ f := &Finder{
repo: repo, repo: repo,
pat: pat, pat: pat,
out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON, printer: printer, stdout: term.OutputRaw()}, out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
printer: printer,
} }
if opts.BlobID { if opts.BlobID {
@@ -675,8 +635,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
} }
} }
var filteredSnapshots []*data.Snapshot var filteredSnapshots []*restic.Snapshot
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots, printer) { for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
filteredSnapshots = append(filteredSnapshots, sn) filteredSnapshots = append(filteredSnapshots, sn)
} }
if ctx.Err() != nil { if ctx.Err() != nil {
@@ -692,7 +652,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts global.Options, args [
for _, sn := range filteredSnapshots { for _, sn := range filteredSnapshots {
if f.blobIDs != nil || f.treeIDs != nil { if f.blobIDs != nil || f.treeIDs != nil {
if err = f.findIDs(ctx, sn); err != nil && !errors.Is(err, errFindDone) { if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
return err return err
} }
continue continue
+6 -149
View File
@@ -3,23 +3,18 @@ package main
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"os"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
) )
func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts global.Options, pattern string) []byte { func testRunFind(t testing.TB, wantJSON bool, opts FindOptions, gopts GlobalOptions, pattern string) []byte {
buf, err := withCaptureStdout(t, gopts, func(ctx context.Context, gopts global.Options) error { buf, err := withCaptureStdout(func() error {
gopts.JSON = wantJSON gopts.JSON = wantJSON
return runFind(ctx, opts, gopts, []string{pattern}, gopts.Term) return runFind(context.TODO(), opts, gopts, []string{pattern})
}) })
rtest.OK(t, err) rtest.OK(t, err)
return buf.Bytes() return buf.Bytes()
@@ -100,7 +95,7 @@ func TestFindSorting(t *testing.T) {
env, cleanup := withTestEnvironment(t) env, cleanup := withTestEnvironment(t)
defer cleanup() defer cleanup()
testSetupBackupData(t, env) datafile := testSetupBackupData(t, env)
opts := BackupOptions{} opts := BackupOptions{}
// first backup // first backup
@@ -119,14 +114,14 @@ func TestFindSorting(t *testing.T) {
// first restic find - with default FindOptions{} // first restic find - with default FindOptions{}
results := testRunFind(t, true, FindOptions{}, env.gopts, "testfile") results := testRunFind(t, true, FindOptions{}, env.gopts, "testfile")
lines := strings.Split(string(results), "\n") lines := strings.Split(string(results), "\n")
rtest.Assert(t, len(lines) == 2, "expected two lines of output, found %d", len(lines)) rtest.Assert(t, len(lines) == 2, "expected two files found in repo (%v), found %d", datafile, len(lines))
matches := []testMatches{} matches := []testMatches{}
rtest.OK(t, json.Unmarshal(results, &matches)) rtest.OK(t, json.Unmarshal(results, &matches))
// run second restic find with --reverse, sort oldest to newest // run second restic find with --reverse, sort oldest to newest
resultsReverse := testRunFind(t, true, FindOptions{Reverse: true}, env.gopts, "testfile") resultsReverse := testRunFind(t, true, FindOptions{Reverse: true}, env.gopts, "testfile")
lines = strings.Split(string(resultsReverse), "\n") lines = strings.Split(string(resultsReverse), "\n")
rtest.Assert(t, len(lines) == 2, "expected two lines of output, found %d", len(lines)) rtest.Assert(t, len(lines) == 2, "expected two files found in repo (%v), found %d", datafile, len(lines))
matchesReverse := []testMatches{} matchesReverse := []testMatches{}
rtest.OK(t, json.Unmarshal(resultsReverse, &matchesReverse)) rtest.OK(t, json.Unmarshal(resultsReverse, &matchesReverse))
@@ -136,141 +131,3 @@ func TestFindSorting(t *testing.T) {
rtest.Assert(t, matches[0].SnapshotID == matchesReverse[1].SnapshotID, "matches should be sorted 1") rtest.Assert(t, matches[0].SnapshotID == matchesReverse[1].SnapshotID, "matches should be sorted 1")
rtest.Assert(t, matches[1].SnapshotID == matchesReverse[0].SnapshotID, "matches should be sorted 2") rtest.Assert(t, matches[1].SnapshotID == matchesReverse[0].SnapshotID, "matches should be sorted 2")
} }
func TestFindInvalidTimeRange(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
err := runFind(context.TODO(), FindOptions{Oldest: "2026-01-01", Newest: "2020-01-01"}, env.gopts, []string{"quack"}, env.gopts.Term)
rtest.Assert(t, err != nil && err.Error() == "Fatal: --oldest must specify a time before --newest",
"unexpected error message: %v", err)
}
// JsonOutput is the struct `restic find --json` produces
type JSONOutput struct {
ObjectType string `json:"object_type"`
ID string `json:"id"`
Path string `json:"path"`
ParentTree string `json:"parent_tree,omitempty"`
SnapshotID string `json:"snapshot"`
Time time.Time `json:"time,omitempty"`
}
func TestFindPackfile(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
// backup
backupPath := env.testdata + "/0/0/9"
testRunBackup(t, "", []string{backupPath}, BackupOptions{}, env.gopts)
sn1 := testListSnapshots(t, env.gopts, 1)[0]
// do all the testing wrapped inside withTermStatus()
err := withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
rtest.OK(t, err)
defer unlock()
// load master index
rtest.OK(t, repo.LoadIndex(ctx, printer))
packID := restic.ID{}
done := false
err = repo.ListBlobs(ctx, func(pb restic.PackedBlob) {
if !done && pb.Type == restic.TreeBlob {
packID = pb.PackID
done = true
}
})
rtest.OK(t, err)
rtest.Assert(t, !packID.IsNull(), "expected a tree packfile ID")
findOptions := FindOptions{PackID: true}
results := testRunFind(t, true, findOptions, env.gopts, packID.String())
// get the json records
jsonResult := []JSONOutput{}
rtest.OK(t, json.Unmarshal(results, &jsonResult))
rtest.Assert(t, len(jsonResult) > 0, "expected at least one tree record in the packfile")
// look at the last record
lastIndex := len(jsonResult) - 1
record := jsonResult[lastIndex]
rtest.Assert(t, record.ObjectType == "tree" && record.SnapshotID == sn1.String(),
"expected a tree record with known snapshot id, but got type=%s and snapID=%s instead of %s",
record.ObjectType, record.SnapshotID, sn1.String())
backupPath = filepath.ToSlash(backupPath)[2:] // take the offending drive mapping away
rtest.Assert(t, strings.Contains(record.Path, backupPath), "expected %q as part of %q", backupPath, record.Path)
return nil
})
rtest.OK(t, err)
}
func TestFindPackID(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
dir009 := filepath.Join(env.testdata, "0", "0", "9")
dirEntries, err := os.ReadDir(dir009)
rtest.OK(t, err)
numberOfFiles := len(dirEntries)
// backup
testRunBackup(t, "", []string{dir009}, BackupOptions{}, env.gopts)
sn1 := testListSnapshots(t, env.gopts, 1)[0]
// extract packfile ID from repository index
dataPackID := restic.ID{}
treePackID := restic.ID{}
err = withTermStatus(t, env.gopts, func(ctx context.Context, gopts global.Options) error {
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, gopts.Term)
_, repo, unlock, err := openWithReadLock(ctx, gopts, false, printer)
rtest.OK(t, err)
defer unlock()
// load Index
rtest.OK(t, repo.LoadIndex(ctx, nil))
// go through all index entries and collect data and tree packfile(s)
rtest.OK(t, repo.ListBlobs(ctx, func(blob restic.PackedBlob) {
switch blob.Type {
case restic.DataBlob:
dataPackID = blob.PackID
case restic.TreeBlob:
treePackID = blob.PackID
}
}))
return nil
})
rtest.OK(t, err)
// look for data packfile
rtest.Assert(t, !dataPackID.IsNull(), "expected to find data packfile in repo")
packID := dataPackID.String()
out := testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID)
findRes := []JSONOutput{}
rtest.OK(t, json.Unmarshal(out, &findRes))
rtest.Assert(t, len(findRes) == numberOfFiles, "expected %d entries for this packfile, got %d",
numberOfFiles, len(findRes))
// look for tree packfile
rtest.Assert(t, !treePackID.IsNull(), "expected to find tree packfile in repo")
packID = treePackID.String()
out = testRunFind(t, true, FindOptions{PackID: true}, env.gopts, packID)
findRes = []JSONOutput{}
rtest.OK(t, json.Unmarshal(out, &findRes))
record := findRes[len(findRes)-1]
rtest.Equals(t, record.ObjectType, "tree")
rtest.Equals(t, record.SnapshotID, sn1.String())
// windows path are messy, so we get rid of the messy bits at the start
// exp: "/C/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9"
// got: "C:/Users/RUNNER~1/AppData/Local/Temp/restic-test-2921201257/testdata/0/0/9"
rtest.Equals(t, filepath.ToSlash(record.Path)[2:], filepath.ToSlash(dir009)[2:])
}
+38 -38
View File
@@ -7,16 +7,14 @@ import (
"io" "io"
"strconv" "strconv"
"github.com/restic/restic/internal/data"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/termstatus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
func newForgetCommand(globalOptions *global.Options) *cobra.Command { func newForgetCommand() *cobra.Command {
var opts ForgetOptions var opts ForgetOptions
var pruneOpts PruneOptions var pruneOpts PruneOptions
@@ -51,8 +49,9 @@ Exit status is 12 if the password is incorrect.
GroupID: cmdGroupDefault, GroupID: cmdGroupDefault,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
finalizeSnapshotFilter(&opts.SnapshotFilter) term, cancel := setupTermstatus()
return runForget(cmd.Context(), opts, pruneOpts, *globalOptions, globalOptions.Term, args) defer cancel()
return runForget(cmd.Context(), opts, pruneOpts, globalOptions, term, args)
}, },
} }
@@ -105,21 +104,21 @@ type ForgetOptions struct {
Weekly ForgetPolicyCount Weekly ForgetPolicyCount
Monthly ForgetPolicyCount Monthly ForgetPolicyCount
Yearly ForgetPolicyCount Yearly ForgetPolicyCount
Within data.Duration Within restic.Duration
WithinHourly data.Duration WithinHourly restic.Duration
WithinDaily data.Duration WithinDaily restic.Duration
WithinWeekly data.Duration WithinWeekly restic.Duration
WithinMonthly data.Duration WithinMonthly restic.Duration
WithinYearly data.Duration WithinYearly restic.Duration
KeepTags data.TagLists KeepTags restic.TagLists
UnsafeAllowRemoveAll bool UnsafeAllowRemoveAll bool
data.SnapshotFilter restic.SnapshotFilter
Compact bool Compact bool
// Grouping // Grouping
GroupBy data.SnapshotGroupByOptions GroupBy restic.SnapshotGroupByOptions
DryRun bool DryRun bool
Prune bool Prune bool
} }
@@ -150,7 +149,7 @@ func (opts *ForgetOptions) AddFlags(f *pflag.FlagSet) {
initMultiSnapshotFilter(f, &opts.SnapshotFilter, false) initMultiSnapshotFilter(f, &opts.SnapshotFilter, false)
f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format") f.BoolVarP(&opts.Compact, "compact", "c", false, "use compact output format")
opts.GroupBy = data.SnapshotGroupByOptions{Host: true, Path: true} opts.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.VarP(&opts.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done") f.BoolVarP(&opts.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
f.BoolVar(&opts.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed") f.BoolVar(&opts.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
@@ -164,7 +163,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
return errors.Fatal("negative values other than -1 are not allowed for --keep-*") return errors.Fatal("negative values other than -1 are not allowed for --keep-*")
} }
for _, d := range []data.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily, for _, d := range []restic.Duration{opts.Within, opts.WithinHourly, opts.WithinDaily,
opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} { opts.WithinMonthly, opts.WithinWeekly, opts.WithinYearly} {
if d.Hours < 0 || d.Days < 0 || d.Months < 0 || d.Years < 0 { if d.Hours < 0 || d.Days < 0 || d.Months < 0 || d.Years < 0 {
return errors.Fatal("durations containing negative values are not allowed for --keep-within*") return errors.Fatal("durations containing negative values are not allowed for --keep-within*")
@@ -174,7 +173,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
return nil return nil
} }
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts global.Options, term ui.Terminal, args []string) error { func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := verifyForgetOptions(&opts) err := verifyForgetOptions(&opts)
if err != nil { if err != nil {
return err return err
@@ -189,17 +188,22 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command") return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
} }
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term) ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock, printer)
if err != nil { if err != nil {
return err return err
} }
defer unlock() defer unlock()
var snapshots data.Snapshots verbosity := gopts.verbosity
if gopts.JSON {
verbosity = 0
}
printer := newTerminalProgressPrinter(verbosity, term)
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet() removeSnIDs := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args, printer) { for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }
if ctx.Err() != nil { if ctx.Err() != nil {
@@ -214,12 +218,12 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
removeSnIDs.Insert(*sn.ID()) removeSnIDs.Insert(*sn.ID())
} }
} else { } else {
snapshotGroups, _, err := data.GroupSnapshots(snapshots, opts.GroupBy) snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
if err != nil { if err != nil {
return err return err
} }
policy := data.ExpirePolicy{ policy := restic.ExpirePolicy{
Last: int(opts.Last), Last: int(opts.Last),
Hourly: int(opts.Hourly), Hourly: int(opts.Hourly),
Daily: int(opts.Daily), Daily: int(opts.Daily),
@@ -254,14 +258,14 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
} }
if gopts.Verbose >= 1 && !gopts.JSON { if gopts.Verbose >= 1 && !gopts.JSON {
err = PrintSnapshotGroupHeader(gopts.Term.OutputWriter(), k) err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
if err != nil { if err != nil {
return err return err
} }
} }
var key data.SnapshotGroupKey var key restic.SnapshotGroupKey
if err := json.Unmarshal([]byte(k), &key); err != nil { if json.Unmarshal([]byte(k), &key) != nil {
return err return err
} }
@@ -270,25 +274,21 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
fg.Host = key.Hostname fg.Host = key.Hostname
fg.Paths = key.Paths fg.Paths = key.Paths
keep, remove, reasons := data.ApplyPolicy(snapshotGroup, policy) keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
if !policy.Empty() && len(keep) == 0 { if !policy.Empty() && len(keep) == 0 {
return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String()) return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String())
} }
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON { if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
printer.P("keep %d snapshots:\n", len(keep)) printer.P("keep %d snapshots:\n", len(keep))
if err := PrintSnapshots(gopts.Term.OutputWriter(), keep, reasons, opts.Compact); err != nil { PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
return err
}
printer.P("\n") printer.P("\n")
} }
fg.Keep = asJSONSnapshots(keep) fg.Keep = asJSONSnapshots(keep)
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON { if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
printer.P("remove %d snapshots:\n", len(remove)) printer.P("remove %d snapshots:\n", len(remove))
if err := PrintSnapshots(gopts.Term.OutputWriter(), remove, nil, opts.Compact); err != nil { PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
return err
}
printer.P("\n") printer.P("\n")
} }
fg.Remove = asJSONSnapshots(remove) fg.Remove = asJSONSnapshots(remove)
@@ -331,7 +331,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
} }
if gopts.JSON && len(jsonGroups) > 0 { if gopts.JSON && len(jsonGroups) > 0 {
err = printJSONForget(gopts.Term.OutputWriter(), jsonGroups) err = printJSONForget(globalOptions.stdout, jsonGroups)
if err != nil { if err != nil {
return err return err
} }
@@ -348,7 +348,7 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs)) printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
} }
pruneOptions.DryRun = opts.DryRun pruneOptions.DryRun = opts.DryRun
return runPruneWithRepo(ctx, pruneOptions, repo, removeSnIDs, printer) return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
} }
return nil return nil
@@ -364,7 +364,7 @@ type ForgetGroup struct {
Reasons []KeepReason `json:"reasons"` Reasons []KeepReason `json:"reasons"`
} }
func asJSONSnapshots(list data.Snapshots) []Snapshot { func asJSONSnapshots(list restic.Snapshots) []Snapshot {
var resultList []Snapshot var resultList []Snapshot
for _, sn := range list { for _, sn := range list {
k := Snapshot{ k := Snapshot{
@@ -383,7 +383,7 @@ type KeepReason struct {
Matches []string `json:"matches"` Matches []string `json:"matches"`
} }
func asJSONKeeps(list []data.KeepReason) []KeepReason { func asJSONKeeps(list []restic.KeepReason) []KeepReason {
var resultList []KeepReason var resultList []KeepReason
for _, keep := range list { for _, keep := range list {
k := KeepReason{ k := KeepReason{
+14 -14
View File
@@ -6,22 +6,22 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/global"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui/termstatus"
) )
func testRunForgetMayFail(t testing.TB, gopts global.Options, opts ForgetOptions, args ...string) error { func testRunForgetMayFail(gopts GlobalOptions, opts ForgetOptions, args ...string) error {
pruneOpts := PruneOptions{ pruneOpts := PruneOptions{
MaxUnused: "5%", MaxUnused: "5%",
} }
return withTermStatus(t, gopts, func(ctx context.Context, gopts global.Options) error { return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
return runForget(context.TODO(), opts, pruneOpts, gopts, gopts.Term, args) return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
}) })
} }
func testRunForget(t testing.TB, gopts global.Options, opts ForgetOptions, args ...string) { func testRunForget(t testing.TB, gopts GlobalOptions, opts ForgetOptions, args ...string) {
rtest.OK(t, testRunForgetMayFail(t, gopts, opts, args...)) rtest.OK(t, testRunForgetMayFail(gopts, opts, args...))
} }
func TestRunForgetSafetyNet(t *testing.T) { func TestRunForgetSafetyNet(t *testing.T) {
@@ -38,27 +38,27 @@ func TestRunForgetSafetyNet(t *testing.T) {
testListSnapshots(t, env.gopts, 2) testListSnapshots(t, env.gopts, 2)
// --keep-tags invalid // --keep-tags invalid
err := testRunForgetMayFail(t, env.gopts, ForgetOptions{ err := testRunForgetMayFail(env.gopts, ForgetOptions{
KeepTags: data.TagLists{data.TagList{"invalid"}}, KeepTags: restic.TagLists{restic.TagList{"invalid"}},
GroupBy: data.SnapshotGroupByOptions{Host: true, Path: true}, GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true},
}) })
rtest.Assert(t, strings.Contains(err.Error(), `refusing to delete last snapshot of snapshot group "host example, path`), "wrong error message got %v", err) rtest.Assert(t, strings.Contains(err.Error(), `refusing to delete last snapshot of snapshot group "host example, path`), "wrong error message got %v", err)
// disallow `forget --unsafe-allow-remove-all` // disallow `forget --unsafe-allow-remove-all`
err = testRunForgetMayFail(t, env.gopts, ForgetOptions{ err = testRunForgetMayFail(env.gopts, ForgetOptions{
UnsafeAllowRemoveAll: true, UnsafeAllowRemoveAll: true,
}) })
rtest.Assert(t, strings.Contains(err.Error(), `--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified`), "wrong error message got %v", err) rtest.Assert(t, strings.Contains(err.Error(), `--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified`), "wrong error message got %v", err)
// disallow `forget` without options // disallow `forget` without options
err = testRunForgetMayFail(t, env.gopts, ForgetOptions{}) err = testRunForgetMayFail(env.gopts, ForgetOptions{})
rtest.Assert(t, strings.Contains(err.Error(), `no policy was specified, no snapshots will be removed`), "wrong error message got %v", err) rtest.Assert(t, strings.Contains(err.Error(), `no policy was specified, no snapshots will be removed`), "wrong error message got %v", err)
// `forget --host example --unsafe-allow-remove-all` should work // `forget --host example --unsafe-allow-remove-all` should work
testRunForget(t, env.gopts, ForgetOptions{ testRunForget(t, env.gopts, ForgetOptions{
UnsafeAllowRemoveAll: true, UnsafeAllowRemoveAll: true,
GroupBy: data.SnapshotGroupByOptions{Host: true, Path: true}, GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true},
SnapshotFilter: data.SnapshotFilter{ SnapshotFilter: restic.SnapshotFilter{
Hosts: []string{opts.Host}, Hosts: []string{opts.Host},
}, },
}) })
+16 -47
View File
@@ -3,7 +3,7 @@ package main
import ( import (
"testing" "testing"
"github.com/restic/restic/internal/data" "github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
@@ -69,18 +69,18 @@ func TestForgetOptionValues(t *testing.T) {
{ForgetOptions{Weekly: -2}, negValErrorMsg}, {ForgetOptions{Weekly: -2}, negValErrorMsg},
{ForgetOptions{Monthly: -2}, negValErrorMsg}, {ForgetOptions{Monthly: -2}, negValErrorMsg},
{ForgetOptions{Yearly: -2}, negValErrorMsg}, {ForgetOptions{Yearly: -2}, negValErrorMsg},
{ForgetOptions{Within: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, {ForgetOptions{Within: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinHourly: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinDaily: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinWeekly: data.ParseDurationOrPanic("1y2m3d3h")}, ""}, {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d3h")}, ""},
{ForgetOptions{WithinMonthly: data.ParseDurationOrPanic("2y4m6d8h")}, ""}, {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""},
{ForgetOptions{WithinYearly: data.ParseDurationOrPanic("2y4m6d8h")}, ""}, {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y4m6d8h")}, ""},
{ForgetOptions{Within: data.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg}, {ForgetOptions{Within: restic.ParseDurationOrPanic("-1y2m3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinHourly: data.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg}, {ForgetOptions{WithinHourly: restic.ParseDurationOrPanic("1y-2m3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinDaily: data.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg}, {ForgetOptions{WithinDaily: restic.ParseDurationOrPanic("1y2m-3d3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinWeekly: data.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg}, {ForgetOptions{WithinWeekly: restic.ParseDurationOrPanic("1y2m3d-3h")}, negDurationValErrorMsg},
{ForgetOptions{WithinMonthly: data.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg}, {ForgetOptions{WithinMonthly: restic.ParseDurationOrPanic("-2y4m6d8h")}, negDurationValErrorMsg},
{ForgetOptions{WithinYearly: data.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg}, {ForgetOptions{WithinYearly: restic.ParseDurationOrPanic("2y-4m6d8h")}, negDurationValErrorMsg},
} }
for _, testCase := range testCases { for _, testCase := range testCases {
@@ -96,38 +96,7 @@ func TestForgetOptionValues(t *testing.T) {
func TestForgetHostnameDefaulting(t *testing.T) { func TestForgetHostnameDefaulting(t *testing.T) {
t.Setenv("RESTIC_HOST", "testhost") t.Setenv("RESTIC_HOST", "testhost")
opts := ForgetOptions{}
tests := []struct { opts.AddFlags(pflag.NewFlagSet("test", pflag.ContinueOnError))
name string rtest.Equals(t, []string{"testhost"}, opts.Hosts)
args []string
want []string
}{
{
name: "env default when flag not set",
args: nil,
want: []string{"testhost"},
},
{
name: "flag overrides env",
args: []string{"--host", "flaghost"},
want: []string{"flaghost"},
},
{
name: "empty flag clears env",
args: []string{"--host", ""},
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
set := pflag.NewFlagSet(tt.name, pflag.ContinueOnError)
opts := ForgetOptions{}
opts.AddFlags(set)
err := set.Parse(tt.args)
rtest.Assert(t, err == nil, "expected no error for input")
finalizeSnapshotFilter(&opts.SnapshotFilter)
rtest.Equals(t, tt.want, opts.Hosts)
})
}
} }
+17 -19
View File
@@ -6,15 +6,12 @@ import (
"time" "time"
"github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/global"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/cobra/doc" "github.com/spf13/cobra/doc"
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
func newGenerateCommand(globalOptions *global.Options) *cobra.Command { func newGenerateCommand() *cobra.Command {
var opts generateOptions var opts generateOptions
cmd := &cobra.Command{ cmd := &cobra.Command{
@@ -22,7 +19,7 @@ func newGenerateCommand(globalOptions *global.Options) *cobra.Command {
Short: "Generate manual pages and auto-completion files (bash, fish, zsh, powershell)", Short: "Generate manual pages and auto-completion files (bash, fish, zsh, powershell)",
Long: ` Long: `
The "generate" command writes automatically generated files (like the man pages The "generate" command writes automatically generated files (like the man pages
and the auto-completion files for bash, fish, powershell and zsh). and the auto-completion files for bash, fish and zsh).
EXIT STATUS EXIT STATUS
=========== ===========
@@ -32,7 +29,7 @@ Exit status is 1 if there was any error.
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error { RunE: func(_ *cobra.Command, args []string) error {
return runGenerate(opts, *globalOptions, args, globalOptions.Term) return runGenerate(opts, args)
}, },
} }
opts.AddFlags(cmd.Flags()) opts.AddFlags(cmd.Flags())
@@ -55,7 +52,7 @@ func (opts *generateOptions) AddFlags(f *pflag.FlagSet) {
f.StringVar(&opts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file` (`-` for stdout)") f.StringVar(&opts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file` (`-` for stdout)")
} }
func writeManpages(root *cobra.Command, dir string, printer progress.Printer) error { func writeManpages(root *cobra.Command, dir string) error {
// use a fixed date for the man pages so that generating them is deterministic // use a fixed date for the man pages so that generating them is deterministic
date, err := time.Parse("Jan 2006", "Jan 2017") date, err := time.Parse("Jan 2006", "Jan 2017")
if err != nil { if err != nil {
@@ -69,12 +66,14 @@ func writeManpages(root *cobra.Command, dir string, printer progress.Printer) er
Date: &date, Date: &date,
} }
printer.P("writing man pages to directory %v", dir) Verbosef("writing man pages to directory %v\n", dir)
return doc.GenManTree(root, header, dir) return doc.GenManTree(root, header, dir)
} }
func writeCompletion(filename string, shell string, generate func(w io.Writer) error, printer progress.Printer, gopts global.Options) (err error) { func writeCompletion(filename string, shell string, generate func(w io.Writer) error) (err error) {
printer.PT("writing %s completion file to %v", shell, filename) if stdoutIsTerminal() {
Verbosef("writing %s completion file to %v\n", shell, filename)
}
var outWriter io.Writer var outWriter io.Writer
if filename != "-" { if filename != "-" {
var outFile *os.File var outFile *os.File
@@ -85,7 +84,7 @@ func writeCompletion(filename string, shell string, generate func(w io.Writer) e
defer func() { err = outFile.Close() }() defer func() { err = outFile.Close() }()
outWriter = outFile outWriter = outFile
} else { } else {
outWriter = gopts.Term.OutputWriter() outWriter = globalOptions.stdout
} }
err = generate(outWriter) err = generate(outWriter)
@@ -111,16 +110,15 @@ func checkStdoutForSingleShell(opts generateOptions) error {
return nil return nil
} }
func runGenerate(opts generateOptions, gopts global.Options, args []string, term ui.Terminal) error { func runGenerate(opts generateOptions, args []string) error {
if len(args) > 0 { if len(args) > 0 {
return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags") return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
} }
printer := ui.NewProgressPrinter(gopts.JSON, gopts.Verbosity, term) cmdRoot := newRootCommand()
cmdRoot := newRootCommand(&global.Options{})
if opts.ManDir != "" { if opts.ManDir != "" {
err := writeManpages(cmdRoot, opts.ManDir, printer) err := writeManpages(cmdRoot, opts.ManDir)
if err != nil { if err != nil {
return err return err
} }
@@ -132,28 +130,28 @@ func runGenerate(opts generateOptions, gopts global.Options, args []string, term
} }
if opts.BashCompletionFile != "" { if opts.BashCompletionFile != "" {
err := writeCompletion(opts.BashCompletionFile, "bash", cmdRoot.GenBashCompletion, printer, gopts) err := writeCompletion(opts.BashCompletionFile, "bash", cmdRoot.GenBashCompletion)
if err != nil { if err != nil {
return err return err
} }
} }
if opts.FishCompletionFile != "" { if opts.FishCompletionFile != "" {
err := writeCompletion(opts.FishCompletionFile, "fish", func(w io.Writer) error { return cmdRoot.GenFishCompletion(w, true) }, printer, gopts) err := writeCompletion(opts.FishCompletionFile, "fish", func(w io.Writer) error { return cmdRoot.GenFishCompletion(w, true) })
if err != nil { if err != nil {
return err return err
} }
} }
if opts.ZSHCompletionFile != "" { if opts.ZSHCompletionFile != "" {
err := writeCompletion(opts.ZSHCompletionFile, "zsh", cmdRoot.GenZshCompletion, printer, gopts) err := writeCompletion(opts.ZSHCompletionFile, "zsh", cmdRoot.GenZshCompletion)
if err != nil { if err != nil {
return err return err
} }
} }
if opts.PowerShellCompletionFile != "" { if opts.PowerShellCompletionFile != "" {
err := writeCompletion(opts.PowerShellCompletionFile, "powershell", cmdRoot.GenPowerShellCompletion, printer, gopts) err := writeCompletion(opts.PowerShellCompletionFile, "powershell", cmdRoot.GenPowerShellCompletion)
if err != nil { if err != nil {
return err return err
} }

Some files were not shown because too many files have changed in this diff Show More